@nexrender/core 1.43.2 → 1.44.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nexrender/core",
-  "version": "1.43.2",
+  "version": "1.44.0",
   "main": "src/index.js",
   "author": "Inlife",
   "scripts": {
@@ -15,8 +15,11 @@
     "match-all": "^1.2.5",
     "mime-types": "^2.1.29",
     "mkdirp": "^1.0.4",
+    "nanoid": "^3.2.0",
+    "posthog-node": "^3.1.1",
     "requireg": "^0.2.1",
-    "rimraf": "^3.0.2"
+    "rimraf": "^3.0.2",
+    "systeminformation": "^5.18.3"
   },
   "peerDependencies": {
     "@nexrender/action-copy": "^1.0.0",
@@ -30,5 +33,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "eb7ee19e97b365296835193daba676a7da1e2a38"
+  "gitHead": "10534b5ae73c920a6ea3c5cc7c3d6ac542112319"
 }

package/readme.md

@@ -71,5 +71,6 @@ Second one is responsible for mainly job-related operations of the full cycle: d
 * `addLicense` - boolean, providing false will disable ae_render_only_node.txt license file auto-creation (true by default)
 * `forceCommandLinePatch` - boolean, providing true will force patch re-installation
 * `onInstanceSpawn` - a callback, if provided, gets called when **aerender** instance is getting spawned, with instance pointer. Can be later used to kill a hung aerender process. Callback signature: `function (instance, job, settings) {}`
+* `noAnalytics` - boolean, enables or disables built-in fully-anonymous analytics, false by default
 * `actions` - an object with keys corresponding to the `module` field when defining an action, value should be a function matching expected signature of an action. Used for defining actions programmatically without needing to package the action as a separate package
 * `cache` - boolean or string. Set the cache folder used by HTTP assets. If `true` will use the default path of `${workpath}/http-cache`, if set to a string it will be interpreted as a filesystem path to the cache folder.

package/src/helpers/analytics.js

@@ -0,0 +1,132 @@
+const fs           = require('fs')
+const os           = require('os')
+const path         = require('path')
+const crypto       = require('crypto')
+
+const si           = require('systeminformation')
+const { nanoid }   = require('nanoid')
+const {PostHog}    = require('posthog-node')
+const childProcess = require('child_process')
+
+const { version }  = require('../../package.json')
+
+const hash = (data, salt) => crypto
+    .createHmac('md5', salt)
+    .update(data)
+    .digest('hex');
+
+const analyticsPublicKey = 'phc_AWcZMlCOqHJiFyFKSoxT9WSrRkdKDFxpiFn8Ww0ZMHu';
+const analytics = new PostHog(analyticsPublicKey, {
+    host: 'https://eu.posthog.com',
+    flushAt: 1,
+    flushInterval: 0,
+    disableGeoip: true,
+});
+
+/**
+ * A helper function to force syncronous tracking
+ *
+ * @param {*} settings
+ * @param {*} event
+ * @param {*} properties
+ */
+const forceSyncRequest = (settings, event, properties) => {
+    const args = JSON.stringify({settings, event, properties})
+    const proc = childProcess.fork(__filename, ['child', args], {
+        stdio: 'ignore',
+    })
+}
+
+/**
+ * Tracking function for analytics
+ *
+ * @param {*} settings
+ * @param {*} event
+ * @param {*} properties
+ * @returns {Promise<void>}
+ */
+const track = async (settings, event, properties = {}, isRemote) => {
+    // if (isRemote) console.log('tracking', event, properties, settings)
+
+    if (settings.noAnalytics === true) return;
+
+    if (!settings.session) {
+        settings.session = hash(nanoid(), 'nexrender-session')
+    }
+
+    // make sure we have a unique id for this user
+    if (!settings.analyticsId) {
+        let filepath = path.join(os.homedir(), '.nexrender', 'uuid')
+        if (fs.existsSync(filepath)) {
+            settings.analyticsId = fs.readFileSync(filepath, 'utf8')
+        } else {
+            settings.analyticsId = hash(nanoid(), 'nexrender-analytics')
+            if (!fs.existsSync(path.dirname(filepath)))
+                fs.mkdirSync(path.dirname(filepath), { recursive: true })
+
+            fs.writeFileSync(filepath, settings.analyticsId)
+        }
+    }
+
+    // make sure we wait to send the event if there is an error
+    if (properties.forced === true) {
+        delete properties.forced
+        properties.$timestamp = (new Date()).toISOString();
+        return forceSyncRequest(settings, event, properties)
+    }
+
+    // collect system info
+    if (!settings.systemInfo) {
+        if (isRemote) console.log('collectiing systeminfo')
+
+        settings.systemInfo = await si.get({
+            cpu: 'manufacturer,brand,cores',
+            mem: 'total',
+            graphics: '*',
+            osInfo: 'platform,arch,distro,release',
+            dockerInfo: 'id',
+        })
+
+        properties.$set_once = {
+            sys_cpu_manufacturer: settings.systemInfo.cpu.manufacturer,
+            sys_cpu_brand: settings.systemInfo.cpu.brand,
+            sys_cpu_cores: settings.systemInfo.cpu.cores,
+            sys_mem_total: settings.systemInfo.mem.total,
+            sys_graphics_vendor: (settings.systemInfo.graphics.controllers[0] || []).vendor,
+            sys_graphics_model: (settings.systemInfo.graphics.controllers[0] || []).model,
+            sys_os_platform: settings.systemInfo.osInfo.platform,
+            sys_os_arch: settings.systemInfo.osInfo.arch,
+            sys_os_distro: settings.systemInfo.osInfo.distro,
+            sys_os_release: settings.systemInfo.osInfo.release,
+            sys_docker: !!settings.systemInfo.dockerInfo.id,
+        }
+    }
+
+    // anonymize job_id
+    if (properties.job_id) {
+        properties.job_id = hash(properties.job_id, settings.analyticsId)
+    }
+
+    const params = {
+        distinctId: settings.analyticsId,
+        event,
+        properties: Object.assign({
+            process: settings.process,
+            version: version,
+            debug: settings.debug,
+            $timestamp: (new Date()).toISOString(),
+            $session_id: settings.session,
+        }, properties)
+    }
+
+    analytics.capture(params);
+    await analytics.flush();
+}
+
+if (process.argv[2] === 'child') {
+    const args = JSON.parse(process.argv[3])
+    track(args.settings, args.event, args.properties, true)
+        .then(() => {})
+}
+
+module.exports = track

package/src/helpers/license.js

@@ -13,15 +13,24 @@ module.exports = (settings) => {
     settings.logger.log(' - ' + nodefile1)
     settings.logger.log(' - ' + nodefile2)
 
+    let applied = false
+
     if (!fs.existsSync(adobe)) {
         fs.mkdirSync(adobe)
     }
 
     if (!fs.existsSync(nodefile1)) {
         fs.writeFileSync(nodefile1, '')
+        applied = true
     }
 
     if (!fs.existsSync(nodefile2)) {
         fs.writeFileSync(nodefile2, '')
+        applied = true
+    }
+
+    if (applied) {
+        settings.track('Init Render License Added')
+        settings.logger.log('added render-only-node licenses for After Effects')
     }
 }

package/src/helpers/patch.js

@@ -39,8 +39,11 @@ module.exports = (settings) => {
         if (patchedMatch[1] !== existingMatch[1]) {
             try {
                 settings.logger.log('out-of-date version of the commandLineRenderer.jsx patch is detected, attepmting to update')
+                settings.track('Init Patch Update Succeeded')
                 writeTo(patched, originalFile)
             } catch (err) {
+                settings.trackSync('Init Patch Update Failed')
+
                 if (err.code == 'EPERM') {
                     settings.logger.log('\n\n              -- E R R O R --\n');
                     settings.logger.log('you need to run application with admin priviledges once');
@@ -62,6 +65,7 @@ module.exports = (settings) => {
 
         if (settings.forceCommandLinePatch) {
             settings.logger.log('forced rewrite of command line patch')
+            settings.track('Init Patch Forced')
             writeTo(patched, originalFile)
         }
     } else {
@@ -75,7 +79,11 @@ module.exports = (settings) => {
             settings.logger.log('patching the command line script')
             fs.chmodSync(originalFile, '755');
             writeTo(patched, originalFile)
+
+            settings.track('Init Patch Install Succeeded')
         } catch (err) {
+            settings.trackSync('Init Patch Install Failed')
+
             if (err.code == 'EPERM') {
                 settings.logger.log('\n\n              -- E R R O R --\n');
                 settings.logger.log('you need to run application with admin priviledges once');

package/src/index.js

@@ -10,6 +10,7 @@ const license      = require('./helpers/license')
 const autofind     = require('./helpers/autofind')
 const patch        = require('./helpers/patch')
 const state        = require('./helpers/state')
+const track        = require('./helpers/analytics')
 
 const setup        = require('./tasks/setup')
 const predownload  = require('./tasks/actions')('predownload')
@@ -36,13 +37,18 @@ if (process.env.NEXRENDER_REQUIRE_PLUGINS) {
     require('@nexrender/provider-sftp');
 }
 
-//
-// https://video.stackexchange.com/questions/16706/rendered-file-with-after-effects-is-very-huge
-//
-
 const init = (settings) => {
     settings = Object.assign({}, settings);
     settings.logger = settings.logger || console;
+    settings.track = (...args) => track(settings, ...args);
+    settings.trackSync = (event, params) => track(settings, event, { forced: true, ...params })
+
+    // set default process name for analytics
+    if (!settings.process) {
+        settings.process = 'nexrender-core';
+    }
+
+    settings.trackSync('Init Started')
 
     // check for WSL
     settings.wsl = isWsl
@@ -51,12 +57,16 @@ const init = (settings) => {
     const binaryUser = settings.binary && fs.existsSync(settings.binary) ? settings.binary : null;
 
     if (!binaryUser && !binaryAuto) {
+        settings.trackSync('Init Failed', { error: 'no_aerender_binary_found' })
         throw new Error('you should provide a proper path to After Effects\' "aerender" binary')
     }
 
     if (binaryAuto && !binaryUser) {
         settings.logger.log('using automatically determined directory of After Effects installation:')
         settings.logger.log(' - ' + binaryAuto)
+        settings.aeBinaryStrategy = 'auto';
+    } else {
+        settings.aeBinaryStrategy = 'manual';
     }
 
     settings = Object.assign({
@@ -82,15 +92,19 @@ const init = (settings) => {
         binary: binaryUser || binaryAuto,
     })
 
+    // try to detect version
+    settings.aeBinaryVersion = (settings.binary.match(/([0-9]{4})\/Support Files/) || [])[1] || 'Unknown';
+
     // make sure we will have absolute path
     if (!path.isAbsolute(settings.workpath)) {
         settings.workpath = path.join(process.cwd(), settings.workpath);
     }
 
     // if WSL, ask user to define Mapping
-    if (settings.wsl && !settings.wslMap)
+    if (settings.wsl && !settings.wslMap) {
+        settings.trackSync('Init Failed', { error: 'wsl_detected_no_map' });
         throw new Error('WSL detected: provide your WSL drive map; ie. "Z"')
-
+    }
 
     // add license helper
     if (settings.addLicense) {
@@ -101,6 +115,24 @@ const init = (settings) => {
     // Scripts/commandLineRenderer.jsx
     patch(settings);
 
+    settings.trackSync('Init Succeeded', {
+        ae_binary_strategy: settings.aeBinaryStrategy,
+        ae_binary_version: settings.aeBinaryVersion,
+        ae_multi_frames: settings.multiFrames,
+        ae_max_memory_percent: !!settings.maxMemoryPercent,
+        ae_image_cache_percent: !!settings.imageCachePercent,
+
+        wsl_is_detected: !!settings.wsl,
+        wsl_map: !!settings.wslMap,
+
+        set_add_license: settings.addLicense,
+        set_force_patch: settings.forceCommandLinePatch,
+        set_skip_cleanup: settings.skipCleanup,
+        set_skip_render: settings.skipRender,
+        set_stop_onerror: settings.stopOnError,
+        set_custom_workpath: settings.workpath !== path.join(os.tmpdir(), 'nexrender'),
+    })
+
     return settings;
 }
 
@@ -130,5 +162,5 @@ const render = (jobConfig, settings = {}) => {
 
 module.exports = {
     init,
-    render
+    render,
 }

package/src/tasks/actions.js

@@ -21,9 +21,15 @@ module.exports = actionType => (job, settings) => {
     settings.logger.log(`[${job.uid}] applying ${actionType} actions...`);
 
     return PromiseSerial((job.actions[actionType] || []).map(action => () => {
-        if(settings.actions && settings.actions[action.module]){
+        settings.track(`Job Action Started`, {
+            job_id: job.uid, // anonymized internally
+            action_type: actionType,
+            action_module: action.module,
+        })
+
+        if (settings.actions && settings.actions[action.module]) {
             return settings.actions[action.module](job, settings, action, actionType);
-        }else{
+        } else {
             return requireg(action.module)(job, settings, action, actionType).catch(err => {
                 return Promise.reject(new Error(`Error loading ${actionType} module ${action.module}: ${err}`));
             });

package/src/tasks/download.js

@@ -1,18 +1,14 @@
 const fs       = require('fs')
 const url      = require('url')
 const path     = require('path')
+const requireg = require('requireg')
 const fetch    = require('make-fetch-happen')
 const uri2path = require('file-uri-to-path')
 const data2buf = require('data-uri-to-buffer')
-const mime = require('mime-types')
+const mime     = require('mime-types')
 const {expandEnvironmentVariables} = require('../helpers/path')
 
-// TODO: redeuce dep size
-const requireg = require('requireg')
-
 const download = (job, settings, asset) => {
-    if (asset.type == 'data') return Promise.resolve();
-
     // eslint-disable-next-line
     const uri = global.URL ? new URL(asset.src) : url.parse(asset.src)
     const protocol = uri.protocol.replace(/:$/, '');
@@ -50,6 +46,13 @@ const download = (job, settings, asset) => {
 
     asset.dest = path.join(job.workpath, destName);
 
+    settings.track('Job Asset Download Started', {
+        job_id: job.uid, // anonymized internally
+        asset_type: asset.type,
+        asset_protocol: protocol,
+        asset_extension: asset.extension,
+    })
+
     switch (protocol) {
         /* built-in handlers */
         case 'data':
@@ -72,7 +75,7 @@ const download = (job, settings, asset) => {
                 path.join(settings.workpath, "http-cache") :
                 settings.cache;
 
-            if(!asset.params) asset.params = {};
+            if (!asset.params) asset.params = {};
             // Asset's own `params.cachePath` takes precedence (including falsy values)
             asset.params.cachePath = Object.hasOwn(asset.params, 'cachePath') ?
                 asset.params.cachePath :

package/src/tasks/render.js

@@ -3,7 +3,7 @@ const path = require('path')
 const {spawn} = require('child_process')
 const {expandEnvironmentVariables, checkForWSL} = require('../helpers/path')
 
-const progressRegex = /([\d]{1,2}:[\d]{2}:[\d]{2}:[\d]{2})\s+(\(\d+\))/gi;
+const progressRegex = /([\d]{1,2}:[\d]{2}:[\d]{2}:[\d]{2})\s+(\(\d+[UL]?\))/gi;
 const durationRegex = /Duration:\s+([\d]{1,2}:[\d]{2}:[\d]{2}:[\d]{2})/gi;
 const startRegex = /Start:\s+([\d]{1,2}:[\d]{2}:[\d]{2}:[\d]{2})/gi;
 const nexrenderErrorRegex = /Error:\s+(nexrender:.*)$/gim;
@@ -150,6 +150,28 @@ Estimated date of change to the new behavior: 2023-06-01.\n`);
         return data;
     }
 
+    settings.track('Job Render Started', {
+        job_id: job.uid, // anonymized internally
+        job_output_module: job.template.outputModule,
+        job_settings_template: job.template.settingsTemplate,
+        job_output_settings: job.template.outputSettings,
+        job_render_settings: job.template.renderSettings,
+        job_frame_start_set: job.template.frameStart !== undefined,
+        job_frame_end_set: job.template.frameEnd !== undefined,
+        job_frame_increment_set: job.template.frameIncrement !== undefined,
+        job_continue_on_missing: job.template.continueOnMissing,
+        job_image_sequence: job.template.imageSequence,
+        job_multi_frames: settings.multiFrames,
+        job_settings_reuse: settings.reuse,
+        job_settings_skip_render: settings.skipRender,
+        job_settings_stop_on_error: settings.stopOnError,
+        job_settings_skip_cleanup: settings.skipCleanup,
+        job_settings_max_memory_percent: !!settings.maxMemoryPercent,
+        job_settings_image_cache_percent: !!settings.imageCachePercent,
+        job_settings_aeparams_set: !!settings['aeParams'],
+        job_settings_max_render_timeout: settings.maxRenderTimeout,
+    })
+
     // spawn process and begin rendering
     return new Promise((resolve, reject) => {
         renderStopwatch = Date.now();
@@ -170,6 +192,7 @@ Estimated date of change to the new behavior: 2023-06-01.\n`);
 
         instance.on('error', err => {
             clearTimeout(timeoutID);
+            settings.trackSync('Job Render Failed', { job_id: job.uid, error: 'aerender_spawn_error' });
             return reject(new Error(`Error starting aerender process: ${err}`));
         });
 
@@ -187,6 +210,7 @@ Estimated date of change to the new behavior: 2023-06-01.\n`);
             const timeout = 1000 * settings.maxRenderTimeout;
             timeoutID = setTimeout(
                 () => {
+                    settings.trackSync('Job Render Failed', { job_id: job.uid, error: 'aerender_timeout' });
                     reject(new Error(`Maximum rendering time exceeded`));
                     instance.kill('SIGINT');
                 },
@@ -206,17 +230,30 @@ Estimated date of change to the new behavior: 2023-06-01.\n`);
                     settings.logger.log(fs.readFileSync(logPath, 'utf8'))
                 }
 
+                settings.trackSync('Job Render Failed', {
+                    job_id: job.uid, // anonymized internally
+                    exit_code: code,
+                    error: 'aerender_exit_code',
+                });
+
                 clearTimeout(timeoutID);
                 return reject(new Error(outputStr || 'aerender.exe failed to render the output into the file due to an unknown reason'));
             }
 
-            settings.logger.log(`[${job.uid}] rendering took ~${(Date.now() - renderStopwatch) / 1000} sec.`);
+            const renderTime = (Date.now() - renderStopwatch) / 1000
+            settings.logger.log(`[${job.uid}] rendering took ~${renderTime} sec.`);
             settings.logger.log(`[${job.uid}] writing aerender job log to: ${logPath}`);
 
             fs.writeFileSync(logPath, outputStr);
 
             /* resolve job without checking if file exists, or its size for image sequences */
             if (settings.skipRender || job.template.imageSequence || ['jpeg', 'jpg', 'png'].indexOf(outputFile) !== -1) {
+                settings.track('Job Render Finished', {
+                    job_id: job.uid, // anonymized internally
+                    job_finish_reason: 'skipped_check',
+                    job_render_time: renderTime,
+                })
+
                 clearTimeout(timeoutID);
                 return resolve(job)
             }
@@ -244,6 +281,7 @@ Estimated date of change to the new behavior: 2023-06-01.\n`);
                     settings.logger.log(fs.readFileSync(logPath, 'utf8'))
                 }
 
+                settings.trackSync('Job Render Failed', { job_id: job.uid, error: 'aerender_output_not_found' });
                 clearTimeout(timeoutID);
                 return reject(new Error(`Couldn't find a result file: ${outputFile}`))
             }
@@ -256,6 +294,12 @@ Estimated date of change to the new behavior: 2023-06-01.\n`);
                 settings.logger.log(`[${job.uid}] Warning: output file size is less than 1000 bytes (${stats.size} bytes), be advised that file is corrupted, or rendering is still being finished`)
             }
 
+            settings.track('Job Render Finished', {
+                job_id: job.uid, // anonymized internally
+                job_finish_reason: 'success',
+                job_render_time: renderTime,
+            });
+
             clearTimeout(timeoutID);
             resolve(job)
         });

package/src/tasks/script.js

@@ -432,6 +432,17 @@ module.exports = (job, settings) => {
     const base = job.workpath;
 
     job.assets.map(asset => {
+        settings.track('Job Script Asset Wrap', {
+            job_id: job.uid, // anonymized internally
+            script_type: asset.type,
+            script_compostion_set: asset.composition !== undefined,
+            script_layer_strat: asset.layerName ? 'name' : 'index',
+            script_value_strat:
+                asset.value !== undefined ? 'value' : // eslint-disable-line no-nested-ternary, multiline-ternary
+                asset.expression !== undefined ? 'expression' : // eslint-disable-line multiline-ternary
+                undefined,
+        })
+
         switch (asset.type) {
             case 'video':
             case 'audio':

package/src/tasks/setup.js

@@ -56,10 +56,17 @@ P.S. to prevent nexrender from removing temp file data, you also can please prov
 
     // setup paths
     job.workpath = path.join(settings.workpath, job.uid);
-    job.output   = job.output || path.join(job.workpath, job.resultname);
+    job.output   = job.template.output || job.output || path.join(job.workpath, job.resultname);
     mkdirp.sync(job.workpath);
 
     settings.logger.log(`[${job.uid}] working directory is: ${job.workpath}`);
 
+    settings.track('Job Setup Succeeded', {
+        job_id: job.uid, // anonymized internally
+        job_set_output_ext: !!job.template.outputExt,
+        job_set_output_module: !!job.template.outputModule,
+        job_set_output_image_sequence: !!job.template.imageSequence,
+    })
+
     return Promise.resolve(job)
 };

package/test/Scripts/Startup/commandLineRenderer.jsx

@@ -0,0 +1 @@
+// Command line renderer for After Effects. (nexrender-patch-v1.0.3)

package/test/aerender

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+console.log('imitating rendering with aerender :p')
+
+console.log('rendering... 0%')
+Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000)
+console.log('rendering... 50%')
+Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000)
+console.log('rendering... 100%')
+
+process.exit(0)

package/test/empty.json

@@ -0,0 +1,13 @@
+{
+    "template": {
+        "src": "data:text/plain,Hello, World!",
+        "composition": "test"
+    },
+    "assets": [
+        {
+            "type": "image",
+            "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==",
+            "layerName": "test"
+        }
+    ]
+}

package/test/index.js

@@ -1,2 +1,28 @@
-// simple test of code syntax
-require('../src')
+// simple test code
+const {render} = require('../src')
+
+process.env.NEXRENDER_ENABLE_AELOG_PROJECT_FOLDER = true
+
+const job = {
+    template: {
+        src: 'data:text/plain,Hello, World!',
+        composition: 'test',
+        output: __dirname + '/index.js',
+    },
+    assets: [
+        {
+            type: 'image',
+            src: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==',
+            layerName: 'test',
+        }
+    ]
+}
+
+const settings = {
+    binary: __dirname + '/aerender',
+    workpath: __dirname + '/workpath',
+}
+
+render(job, settings)
+    .then(() => console.log('rendered successfully'))
+    .catch(console.error)