@platforma-sdk/r-builder 0.0.2 → 0.0.4

package/package.json

@@ -2,7 +2,7 @@
   "name": "@platforma-sdk/r-builder",
   "description": "Build R run environment for Platforma",
   "type": "module",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "files": [
     "assets/",
     "scripts/"
@@ -12,12 +12,10 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "undici": "~7.2.3",
+    "undici": "~7.4.0",
     "fs-extra": "^11.2.0",
     "tar": "^7.4.3",
-    "unzipper": "^0.12.3",
-    "winston": "^3.17.0",
-    "xz": "^2.0.2"
+    "winston": "^3.17.0"
   },
   "devDependencies": {
     "@milaboratories/eslint-config": "^1.0.1"

package/scripts/builder.js

@@ -22,12 +22,6 @@ import * as windows from './lib/windows.js';
 
 export async function buildRDist(logger, version) {
   const currentOS = os.platform();
-  
-  const doneFile = `${version}.build-done`
-  if (fs.existsSync(doneFile)) {
-    logger.info(`Build of R is already done in this environment.\nTo force rebuild remove '${path.resolve(doneFile)}' file and run again.`)
-    return
-  }
 
   switch (currentOS) {
     case 'darwin': {
@@ -50,8 +44,6 @@ export async function buildRDist(logger, version) {
       process.exit(1);
     }
   }
-
-  fs.writeFileSync(doneFile, '');
 }
 
 export async function buildRDeps(logger, version) {

package/scripts/lib/linux.js

@@ -7,6 +7,39 @@ import * as util from './util.js';
 
 const platformName = os.arch() === 'arm64' ? 'linux-aarch64' : `linux-x64`;
 
+function useCaching(logger) {
+  if (util.isOK('which ccache')) {
+    useCCache(logger)
+  } else if (util.isOK('which sccache')) {
+    useSCCache(logger)
+  } else {
+    logger.warn('No caching tool found. Builds will be slow.')
+  }
+}
+
+/**
+ * Setup current process environment to integrate with sccache
+ *
+ * @param {winston.Logger} logger
+ */
+function useSCCache(logger) {
+  logger.info("Using sccache as build caching tool...")
+  process.env.SCCACHE_GHA_ENABLED = "true"
+  process.env.CC = `sccache gcc`
+  process.env.CXX = `sccache g++`
+}
+
+/**
+ * Setup current process environment to integrate with ccache
+ *
+ * @param {winston.Logger} logger
+ */
+function useCCache(logger) {
+  logger.info("Using ccache as build caching tool...")
+  process.env.CC = `ccache gcc`
+  process.env.CXX = `ccache g++`
+}
+
 async function installSystemPackages(logger) {
   const originalAptSources = '/etc/apt/sources.list';
   logger.info(`Analyzing '${originalAptSources}'`);
@@ -224,19 +257,6 @@ function patchBinElf(logger, binaryFile, rpath) {
   util.runInherit(`patchelf --remove-rpath '${binaryFile}'`);
   util.runInherit(`patchelf --set-rpath '${rpath}' '${binaryFile}'`);
 }
-/**
- * Setup current process environment to integrate with sccache
- *
- * @param {winston.Logger} logger
- */
-function useSCCache(logger) {
-  if (util.isOK('which sccache')) {
-    logger.info("Using sccache as build caching tool...")
-    process.env.SCCACHE_GHA_ENABLED = "true"
-    process.env.CC = `sccache gcc`
-    process.env.CXX = `sccache g++`
-  }
-}
 
 /**
  * Build portable R distribution and all required packages from sources
@@ -250,7 +270,7 @@ export async function buildR(logger, version) {
     process.exit(1);
   }
 
-  useSCCache(logger)
+  useCaching(logger)
 
   const distDir = path.join(util.rDistDir, platformName);
 
@@ -267,6 +287,7 @@ export async function buildR(logger, version) {
 
   fs.renameSync(path.join(distDir, 'bin/R'), path.join(distDir, 'bin/R.orig'))
   fs.copyFileSync(pkg.asset('R.linux.sh'), path.join(distDir, 'bin/R'));
+  fs.chmodSync(path.join(distDir, 'bin/R'), 0o755)
 
   const gccVersion = getGCCMajorVersion()
   const gccUtilsPath = path.join('/usr/lib/gcc/x86_64-linux-gnu', gccVersion.toString())
@@ -280,7 +301,7 @@ export async function buildDeps(logger, version) {
     process.exit(1);
   }
 
-  useSCCache(logger);
+  useCaching(logger);
 
   const distDir = path.join(util.rDistDir, platformName);
 

package/scripts/lib/macos.js

@@ -52,6 +52,7 @@ export async function buildR(logger, version) {
 
   fs.renameSync(path.join(distDir, 'bin/R'), path.join(distDir, 'bin/R.orig'))
   fs.copyFileSync(pkg.asset('R.macos.sh'), path.join(distDir, 'bin/R'))
+  fs.chmodSync(path.join(distDir, 'bin/R'), 0o755)
 
   util.installBasicRPackages(logger, distDir);
 }

package/scripts/lib/util.js

@@ -8,7 +8,6 @@ import { fetch } from 'undici';
 import * as tar from 'tar';
 import winston from 'winston';
 import zlib from 'zlib';
-import xz from 'xz';
 
 import * as pkg from './pkg.js';
 
@@ -20,7 +19,10 @@ export const downloadsDir = `${buildDir}/artifacts/dld`;
 export const artifactsDir = `${buildDir}/artifacts/pkg`;
 
 export const dependenciesDir = 'dependencies'; // where to search for dependencies specification (init.R or renv.lock)
-const renvDirName = 'renv-root' // directory with all renv cache stored inside final R run environment distribution
+
+// directory with all renv cache stored inside final R run environment distribution.
+// When changing this name, remember to update R preparation steps in workflow-tengo SDK.
+const renvDirName = 'renv-root'
 
 const pipelineAsync = promisify(pipeline);
 
@@ -134,15 +136,10 @@ export async function download(logger, url, destination) {
 }
 
 export async function extractTarXz(archivePath, destination) {
-  const archiveStream = fs.createReadStream(archivePath);
-
-  const decompress = new xz.Decompressor();
-
-  await pipelineAsync(
-    archiveStream,
-    decompress,
-    tar.extract({ cwd: destination })
-  );
+  if (!fs.existsSync(destination)) {
+    fs.mkdirSync(destination, { recursive: true });
+  }
+  runInherit(`tar -xJf '${archivePath}' -C '${destination}'`);
 }
 
 export async function extractTarGz(archivePath, destination) {
@@ -234,7 +231,9 @@ export function createLogger(level = 'debug') {
 
     format: winston.format.combine(
       winston.format.printf(({ level, message }) => {
-        const indent = ' '.repeat(level.length + 2); // For ': ' after the level
+        const now = new Date();
+        const time = `[${now.getHours().toString().padStart(2, '0')}:${now.getMinutes().toString().padStart(2, '0')}:${now.getSeconds().toString().padStart(2, '0')}]`;
+        const indent = ' '.repeat(time.length + 1 + level.length + 2); // For ': ' after the level
         const indentedMessage = message
           .split('\n')
           .map((line, index) => (index === 0 ? line : indent + line))
@@ -242,7 +241,7 @@ export function createLogger(level = 'debug') {
 
         const colorize = (l) => winston.format.colorize().colorize(l, l);
 
-        return `${colorize(level)}: ${indentedMessage}`;
+        return `${time} ${colorize(level)}: ${indentedMessage}`;
       })
     ),
 
@@ -270,7 +269,7 @@ export function runR(logger, rRoot, args, opts = {}) {
   const cmdToRun = `${path.resolve(rRoot, 'bin/R')} ${args.join(' ')}`;
   const renvRoot = path.join(path.resolve(rRoot), renvDirName)
 
-  env = {
+  const env = {
     RHOME: path.resolve(rRoot),
     R_HOME_DIR: path.resolve(rRoot),
     RENV_PATHS_ROOT: renvRoot,
@@ -403,22 +402,25 @@ export function buildRDependencies(
       '--no-echo',
       '-e', ...quote(`renv::restore( clean = TRUE )`),
     ], {
-      wd: tempRenvDir,
+      wd: depsDir,
       paths: additionalPaths,
     })
 
+  } else if (fs.existsSync(path.join(depsDir, initFile))) {
+    runInRenv(logger, rRoot, ['--no-echo', `--file=${initFile}`], {
+      wd: depsDir,
+      paths: additionalPaths,
+    })
+
+    if (!fs.existsSync(path.join(depsDir, lockFile))) {
+      runInRenv(logger, rRoot, ['--no-echo', '-e', ...quote(`renv::snapshot()`)])
+    }
+
   } else {
-    if (!fs.existsSync(path.join(depsDir, initFile))) {
       throw new Error(
         `directory ${depsDir} has no ${lockFile} or ${initFile}. No dependencies detected.\n`+
         `Create empty init.R file if you really need empty R run environment without any preinstalled dependencies`
       )
-    }
-
-    runInRenv(logger, rRoot, ['--no-echo', `--file=${initFile}`], {
-      wd: tempRenvDir,
-      paths: additionalPaths,
-    })
   }
 
   const lockData = fs.readFileSync(lockPath, { encoding: 'utf-8' });
@@ -433,8 +435,12 @@ export function buildRDependencies(
     }
   }
 
-  fs.rmSync(tempRenvDir, { recursive: true })
+  // Store resulting 'renv.lock' file in R installation root directory,
+  // so we can use it when preparing renv on Platforma Backend side (workflow-tengo SDK).
+  fs.copyFileSync(path.join(depsDir, lockFile), path.join(rRoot, renvDirName, lockFile))
+
   if (fs.existsSync(origProjectsFile)) {
+    // Restore original 'projects' file to not leave traces of environment preparation.
     fs.renameSync(origProjectsFile, projectsFile)
   }
 
@@ -474,6 +480,8 @@ function patchREnviron(
   envs = {},
   comment = "",
 ) {
+  logger.info(`Patching Renviron file in ${rRoot} directory`)
+
   let rEnvironPath = path.join(rRoot, 'etc', 'Renviron')
   if (!fs.existsSync(rEnvironPath)) {
     rEnvironPath = path.join(rRoot, 'etc', 'Renviron.site')
@@ -487,20 +495,7 @@ function patchREnviron(
   }
 
   for (const [envName, envValue] of Object.entries(envs)) {
-    line = `${envName}=\${${envName}:-'${envValue}'}`
+    const line = `${envName}=\${${envName}:-'${envValue}'}`
     fs.appendFileSync(rEnvironPath, line + '\n', 'utf8');
   }
 }
-
-/**
- * Recreate directory by removing it and creating back.
- *
- * @param {string} dir - path to directory
- */
-function recreateDir(dir) {
-  if (fs.existsSync(dir)) {
-    fs.rmSync(dir, { recursive: true })
-  }
-  fs.mkdirSync(dir, { recursive: true })
-
-}

package/scripts/lib/windows.js

@@ -1,6 +1,6 @@
 import path from 'node:path';
 import os from 'node:os';
-import fs from 'node:fs';
+import fsextra from 'fs-extra';
 
 import * as pkg from './pkg.js';
 import * as util from './util.js';
@@ -17,12 +17,7 @@ export async function buildR(logger, version) {
 
   // Copy Windows R stub files to dist directory
   const stubDir = pkg.asset('windows-x64-stub');
-  await fs.promises.cp(stubDir, distDir, {
-    recursive: true,
-    preserveTimestamps: true,
-    force: true,
-    errorOnExist: false
-  });
+  await fsextra.copy(stubDir, distDir);
 
   // TODO: R for windows
 }