@resolveio/server-lib 22.3.219 → 22.3.221

package/src/util/report-builder-unwinds.ts

@@ -0,0 +1,180 @@
+export function normalizeLookupField(path: string) {
+	if (!path) {
+		return path;
+	}
+
+	let cleaned = path.replace(/\.\$/g, '');
+	cleaned = cleaned.replace(/^[^\.]*\(.*?Lookup.*?\)\.?/, '');
+
+	return cleaned;
+}
+
+function buildSafeSizeExpression(sourceExpr: string) {
+	return {$size: {$cond: [{$isArray: sourceExpr}, sourceExpr, []]}};
+}
+
+export function ensureLookupLocalKeyUnwinds(options: {query: any[]; sizes: string[]; lookupLocalKey: string; lookupAs?: string}) {
+	const {query, sizes, lookupLocalKey, lookupAs = ''} = options || {};
+
+	if (!lookupLocalKey || typeof lookupLocalKey !== 'string') {
+		return;
+	}
+
+	if (!query || !Array.isArray(query) || !sizes || !Array.isArray(sizes)) {
+		return;
+	}
+
+	let unwindPathSeed = lookupLocalKey;
+
+	// Some lookup local keys are stored without `.$` even when sourced from array paths.
+	// Infer the array segment from lookup alias and synthesize the unwind marker.
+	if (!unwindPathSeed.includes('.$') && typeof lookupAs === 'string' && lookupAs.includes('.$.')) {
+		const inferredArrayPrefix = lookupAs.split('.$.')[0];
+		if (inferredArrayPrefix) {
+			if (unwindPathSeed.startsWith(inferredArrayPrefix + '.')) {
+				unwindPathSeed = inferredArrayPrefix + '.$.' + unwindPathSeed.substring(inferredArrayPrefix.length + 1);
+			}
+			else {
+				unwindPathSeed = inferredArrayPrefix + '.$';
+			}
+		}
+	}
+
+	if (!unwindPathSeed.includes('.$')) {
+		return;
+	}
+
+	const fieldData = unwindPathSeed.split('.$');
+	let fieldPath = '';
+
+	for (let i = 0; i < fieldData.length - 1; i++) {
+		fieldPath += fieldData[i];
+		const sizeKey = fieldPath.replace(/\./g, '_');
+		if (!sizes.includes(sizeKey)) {
+			sizes.push(sizeKey);
+		}
+
+		if (!query.some(a => a && a.$addFields && Object.keys(a.$addFields).includes('size_' + sizeKey))) {
+			const sourceExpr = '$' + fieldPath;
+			query.push({$addFields: {['size_' + sizeKey]: buildSafeSizeExpression(sourceExpr)}});
+		}
+
+		const unwindPath = '$' + fieldPath;
+		if (!query.some(a => a && a.$unwind && a.$unwind.path === unwindPath)) {
+			query.push({'$unwind': {path: unwindPath, preserveNullAndEmptyArrays: true}});
+		}
+	}
+}
+
+export function applyLinkFieldUnwinds(options: {
+	query: any[];
+	sizes: string[];
+	fieldsLink: any[];
+	selectedFields?: any[];
+	filterArrayFields?: any[];
+	groupsRow?: any[];
+}) {
+	const {query, sizes, fieldsLink, selectedFields = [], filterArrayFields = [], groupsRow = []} = options || {};
+
+	if (!query || !Array.isArray(query) || !sizes || !Array.isArray(sizes) || !Array.isArray(fieldsLink) || !fieldsLink.length) {
+		return;
+	}
+
+	const leafByPath = new Map<string, any>();
+
+	(selectedFields || []).forEach(field => {
+		if (field?.fieldPath && !leafByPath.has(field.fieldPath)) {
+			leafByPath.set(field.fieldPath, field);
+		}
+	});
+
+	(filterArrayFields || []).forEach(field => {
+		if (field?.fieldPath && !leafByPath.has(field.fieldPath)) {
+			leafByPath.set(field.fieldPath, field);
+		}
+	});
+
+	(groupsRow || []).forEach(group => {
+		if (group?.treeItem?.fieldPath && !leafByPath.has(group.treeItem.fieldPath)) {
+			leafByPath.set(group.treeItem.fieldPath, group.treeItem);
+		}
+	});
+
+	const hasUnwindPath = (unwindPath: string) => {
+		return query.some(a => a && a.$unwind && a.$unwind.path === unwindPath);
+	};
+
+	const ensureUnwindsForPath = (targetPath: string) => {
+		if (!targetPath || typeof targetPath !== 'string' || !targetPath.includes('.$')) {
+			return;
+		}
+
+		const leaf = leafByPath.get(targetPath);
+
+		const fieldData = targetPath.split('.$');
+		let fieldPath = '';
+
+		for (let i = 0; i < fieldData.length - 1; i++) {
+			fieldPath += fieldData[i];
+
+			const lastSegment = fieldPath.split('.$')[fieldPath.split('.$').length - 1];
+			const hasLookup = (lastSegment || '').includes('(Lookup') || (leaf?.lookup_as && fieldPath === leaf.lookup_as);
+
+			if (hasLookup && leaf?.lookup_collection && leaf?.lookup_as) {
+				ensureLookupLocalKeyUnwinds({
+					query,
+					sizes,
+					lookupLocalKey: leaf.lookup_local_key || '',
+					lookupAs: leaf.lookup_as || ''
+				});
+
+				const lookupAsRaw = leaf.lookup_as || '';
+				const lookupAs = lookupAsRaw.replace(/\.\$/g, '');
+				const hasLookupStage = query.some(a => a && a.$lookup && a.$lookup.as === lookupAs);
+
+				if (!hasLookupStage) {
+					query.push({$lookup: {
+						from: leaf.lookup_collection,
+						localField: normalizeLookupField(leaf.lookup_local_key || ''),
+						foreignField: normalizeLookupField(leaf.lookup_foreign_key || ''),
+						as: lookupAs
+					}});
+				}
+
+				const sizeKey = lookupAsRaw.replace(/\.\$\./g, '_').replace(/\./g, '_');
+				if (!query.some(a => a && a.$addFields && Object.keys(a.$addFields).includes('size_' + sizeKey))) {
+					sizes.push(sizeKey);
+					const sourceExpr = '$' + lookupAs;
+					query.push({$addFields: {['size_' + sizeKey]: buildSafeSizeExpression(sourceExpr)}});
+				}
+
+				const unwindPath = '$' + lookupAs;
+				if (!hasUnwindPath(unwindPath)) {
+					query.push({'$unwind': {path: unwindPath, preserveNullAndEmptyArrays: true}});
+				}
+			}
+			else {
+				const sizeKey = fieldPath.replace(/\.\$\./g, '.').replace(/\./g, '_');
+				if (!query.some(a => a && a.$addFields && Object.keys(a.$addFields).includes('size_' + sizeKey))) {
+					sizes.push(sizeKey);
+					const sourceExpr = '$' + fieldPath.replace(/\.\$\./g, '.');
+					query.push({$addFields: {['size_' + sizeKey]: buildSafeSizeExpression(sourceExpr)}});
+				}
+
+				const unwindPath = '$' + fieldPath.replace(/\.\$\./g, '.');
+				if (!hasUnwindPath(unwindPath)) {
+					query.push({'$unwind': {path: unwindPath, preserveNullAndEmptyArrays: true}});
+				}
+			}
+
+			fieldPath += '.$';
+		}
+	};
+
+	(fieldsLink || [])
+		.filter(a => a && a.field_first && a.field_second)
+		.forEach(link => {
+			ensureUnwindsForPath(link.field_first);
+			ensureUnwindsForPath(link.field_second);
+		});
+}

package/src/util/runner-process-janitor.ts

@@ -0,0 +1,219 @@
+export interface RunnerProcessJanitorManifestInput {
+	jobId?: string;
+	ownerId?: string;
+	runnerToken?: string;
+	runnerRunId?: string;
+	runnerGeneration?: string | number;
+	workspaceRoot?: string;
+	projectRoot?: string;
+	clientPort?: string | number;
+	serverPort?: string | number;
+	mongoPort?: string | number;
+	inspectPort?: string | number;
+}
+
+export interface RunnerProcessJanitorScriptOptions {
+	mode?: 'support' | 'runner';
+	staleTtlSeconds?: number;
+	minAvailableMemoryMb?: number;
+	minAvailableDiskMb?: number;
+}
+
+export class RunnerProcessJanitor {
+	public static readonly defaultStaleTtlSeconds = 600;
+	public static readonly defaultMinAvailableMemoryMb = 512;
+	public static readonly defaultMinAvailableDiskMb = 1536;
+
+	public static buildManifest(input: RunnerProcessJanitorManifestInput = {}): Record<string, unknown> {
+		return {
+			job_id: String(input.jobId || '').trim(),
+			owner_id: String(input.ownerId || '').trim(),
+			runner_token: String(input.runnerToken || '').trim(),
+			runner_run_id: String(input.runnerRunId || '').trim(),
+			runner_generation: String(input.runnerGeneration || '').trim(),
+			workspace_root: String(input.workspaceRoot || '').trim(),
+			project_root: String(input.projectRoot || '').trim(),
+			ports: {
+				client: String(input.clientPort || '').trim(),
+				server: String(input.serverPort || '').trim(),
+				mongo: String(input.mongoPort || '').trim(),
+				inspect: String(input.inspectPort || '').trim()
+			},
+			started_at: new Date().toISOString(),
+			heartbeat_at: new Date().toISOString(),
+			cleanup_status: 'running'
+		};
+	}
+
+	public static buildShellLibrary(options: RunnerProcessJanitorScriptOptions = {}): string {
+		const mode = options.mode || 'runner';
+		const staleTtlSeconds = Number.isFinite(Number(options.staleTtlSeconds)) && Number(options.staleTtlSeconds) > 0
+			? Number(options.staleTtlSeconds)
+			: RunnerProcessJanitor.defaultStaleTtlSeconds;
+		const minAvailableMemoryMb = Number.isFinite(Number(options.minAvailableMemoryMb)) && Number(options.minAvailableMemoryMb) > 0
+			? Number(options.minAvailableMemoryMb)
+			: RunnerProcessJanitor.defaultMinAvailableMemoryMb;
+		const minAvailableDiskMb = Number.isFinite(Number(options.minAvailableDiskMb)) && Number(options.minAvailableDiskMb) > 0
+			? Number(options.minAvailableDiskMb)
+			: RunnerProcessJanitor.defaultMinAvailableDiskMb;
+		const prefix = mode === 'support' ? 'RESOLVEIO_SUPPORT_QA' : 'RESOLVEIO_RUNNER_QA';
+		const altPrefix = mode === 'support' ? 'RESOLVEIO_RUNNER_QA' : 'RESOLVEIO_SUPPORT_QA';
+		return [
+			`RUNNER_JANITOR_STALE_TTL_SECONDS="\${${prefix}_STALE_TTL_SECONDS:-\${${altPrefix}_STALE_TTL_SECONDS:-${staleTtlSeconds}}}"`,
+			`RUNNER_JANITOR_MIN_AVAILABLE_MEMORY_MB="\${${prefix}_MIN_AVAILABLE_MEMORY_MB:-\${${altPrefix}_MIN_AVAILABLE_MEMORY_MB:-${minAvailableMemoryMb}}}"`,
+			`RUNNER_JANITOR_MIN_AVAILABLE_DISK_MB="\${${prefix}_MIN_AVAILABLE_DISK_MB:-\${${altPrefix}_MIN_AVAILABLE_DISK_MB:-${minAvailableDiskMb}}}"`,
+			'RUNNER_JANITOR_HEARTBEAT_PID=""',
+			'janitor_json_escape() { node -e "process.stdout.write(JSON.stringify(process.argv[1] || \'\'))" "$1"; }',
+			'janitor_now_ms() { node -e "console.log(Date.now())"; }',
+			'janitor_file_mtime_ms() { node -e "const fs=require(\'fs\'); try{console.log(Math.floor(fs.statSync(process.argv[1]).mtimeMs||0))}catch(e){console.log(0)}" "$1"; }',
+			'janitor_bounded() {',
+			'  local seconds="$1"',
+			'  shift || return 0',
+			'  if command -v timeout >/dev/null 2>&1; then',
+			'    timeout -k 2 "$seconds" "$@" 2>/dev/null || true',
+			'  else',
+			'    "$@" 2>/dev/null || true',
+			'  fi',
+			'}',
+			'janitor_manifest_path() { echo "$ARTIFACT_DIR/runner-process-manifest.json"; }',
+			'janitor_status_path() { echo "$ARTIFACT_DIR/runner-process-cleanup.json"; }',
+			'janitor_runner_token() {',
+			`  local token="\${${prefix}_RUNNER_TOKEN:-\${${altPrefix}_RUNNER_TOKEN:-}}"`,
+			'  if [ -z "$token" ]; then token="$$-$(date +%s)-$RANDOM"; fi',
+			'  echo "$token"',
+			'}',
+			'janitor_write_manifest() {',
+			'  local manifest token job owner run_id generation',
+			'  manifest="$(janitor_manifest_path)"',
+			'  token="$(janitor_runner_token)"',
+			`  job="\${${prefix}_JOB_ID:-\${${altPrefix}_JOB_ID:-}}"`,
+			`  owner="\${${prefix}_OWNER_ID:-\${${altPrefix}_OWNER_ID:-}}"`,
+			`  run_id="\${${prefix}_RUNNER_RUN_ID:-\${${altPrefix}_RUNNER_RUN_ID:-}}"`,
+			`  generation="\${${prefix}_RUNNER_GENERATION:-\${${altPrefix}_RUNNER_GENERATION:-}}"`,
+			'  cat > "$manifest" <<EOF',
+			'{',
+			'  "job_id": $(janitor_json_escape "$job"),',
+			'  "owner_id": $(janitor_json_escape "$owner"),',
+			'  "runner_token": $(janitor_json_escape "$token"),',
+			'  "runner_run_id": $(janitor_json_escape "$run_id"),',
+			'  "runner_generation": $(janitor_json_escape "$generation"),',
+			'  "pid": $$,',
+			'  "workspace_root": $(janitor_json_escape "${REPO_ROOT:-}"),',
+			'  "project_root": $(janitor_json_escape "$PROJECT_ROOT"),',
+			'  "ports": {',
+			'    "client": $(janitor_json_escape "$CLIENT_PORT"),',
+			'    "server": $(janitor_json_escape "$SERVER_PORT"),',
+			'    "mongo": $(janitor_json_escape "$MONGO_PORT"),',
+			'    "inspect": $(janitor_json_escape "$INSPECT_PORT")',
+			'  },',
+			'  "started_at": $(janitor_json_escape "$(date -u +%Y-%m-%dT%H:%M:%SZ)"),',
+			'  "heartbeat_at": $(janitor_json_escape "$(date -u +%Y-%m-%dT%H:%M:%SZ)"),',
+			'  "cleanup_status": "running"',
+			'}',
+			'EOF',
+			'}',
+			'janitor_update_manifest_status() {',
+			'  local status="$1"',
+			'  local manifest',
+			'  manifest="$(janitor_manifest_path)"',
+			'  [ -f "$manifest" ] || return 0',
+			'  node - "$manifest" "$status" <<\'JANITOR_UPDATE_STATUS\'',
+			'const fs = require("fs");',
+			'const [file, status] = process.argv.slice(2);',
+			'try {',
+			'  const data = JSON.parse(fs.readFileSync(file, "utf8"));',
+			'  data.heartbeat_at = new Date().toISOString();',
+			'  data.cleanup_status = status;',
+			'  fs.writeFileSync(file, JSON.stringify(data, null, 2));',
+			'} catch (error) {}',
+			'JANITOR_UPDATE_STATUS',
+			'}',
+			'janitor_update_lock_heartbeat() {',
+			'  local lock_json="${LOCK_DIR:-}/lock.json"',
+			'  [ -f "$lock_json" ] || return 0',
+			'  node - "$lock_json" <<\'JANITOR_UPDATE_LOCK\'',
+			'const fs = require("fs");',
+			'const [file] = process.argv.slice(2);',
+			'try {',
+			'  const data = JSON.parse(fs.readFileSync(file, "utf8"));',
+			'  data.heartbeat_at = new Date().toISOString();',
+			'  fs.writeFileSync(file, JSON.stringify(data, null, 2));',
+			'} catch (error) {}',
+			'JANITOR_UPDATE_LOCK',
+			'}',
+			'janitor_start_heartbeat() {',
+			'  local manifest',
+			'  manifest="$(janitor_manifest_path)"',
+			'  ( while true; do janitor_update_manifest_status running; janitor_update_lock_heartbeat; sleep 15; done ) >/dev/null 2>&1 &',
+			'  RUNNER_JANITOR_HEARTBEAT_PID="$!"',
+			'}',
+			'janitor_stop_heartbeat() {',
+			'  [ -n "$RUNNER_JANITOR_HEARTBEAT_PID" ] || return 0',
+			'  kill "$RUNNER_JANITOR_HEARTBEAT_PID" >/dev/null 2>&1 || true',
+			'  RUNNER_JANITOR_HEARTBEAT_PID=""',
+			'}',
+			'janitor_lock_is_live() {',
+			'  local lock_json="$LOCK_DIR/lock.json"',
+			'  [ -f "$lock_json" ] || return 1',
+			'  node - "$lock_json" "$RUNNER_JANITOR_STALE_TTL_SECONDS" <<\'JANITOR_LOCK_LIVE\'',
+			'const fs = require("fs");',
+			'const [file, ttlRaw] = process.argv.slice(2);',
+			'const ttlMs = Math.max(1, Number(ttlRaw || 600)) * 1000;',
+			'try {',
+			'  const data = JSON.parse(fs.readFileSync(file, "utf8"));',
+			'  const pid = Number(data.pid || 0);',
+			'  const heartbeat = Date.parse(data.heartbeat_at || data.started_at || "");',
+			'  if (!pid || !heartbeat || Date.now() - heartbeat > ttlMs) process.exit(1);',
+			'  process.kill(pid, 0);',
+			'  process.exit(0);',
+			'} catch (error) { process.exit(1); }',
+			'JANITOR_LOCK_LIVE',
+			'}',
+			'janitor_write_lock() {',
+			'  cat > "$LOCK_DIR/lock.json" <<EOF',
+			'{',
+			'  "pid": $$,',
+			'  "runner_token": $(janitor_json_escape "$(janitor_runner_token)"),',
+			'  "runner_run_id": $(janitor_json_escape "${RESOLVEIO_RUNNER_QA_RUNNER_RUN_ID:-${RESOLVEIO_SUPPORT_QA_RUNNER_RUN_ID:-}}"),',
+			'  "runner_generation": $(janitor_json_escape "${RESOLVEIO_RUNNER_QA_RUNNER_GENERATION:-${RESOLVEIO_SUPPORT_QA_RUNNER_GENERATION:-}}"),',
+			'  "project_root": $(janitor_json_escape "$PROJECT_ROOT"),',
+			'  "manifest": $(janitor_json_escape "$(janitor_manifest_path)"),',
+			'  "started_at": $(janitor_json_escape "$(date -u +%Y-%m-%dT%H:%M:%SZ)"),',
+			'  "heartbeat_at": $(janitor_json_escape "$(date -u +%Y-%m-%dT%H:%M:%SZ)")',
+			'}',
+			'EOF',
+			'}',
+			'janitor_check_resources() {',
+			'  local available_mb disk_mb',
+			'  available_mb="$(awk \'/MemAvailable:/ {print int($2/1024)}\' /proc/meminfo 2>/dev/null || echo 999999)"',
+			'  disk_mb="$(df -Pm "$PROJECT_ROOT" 2>/dev/null | awk \'NR==2 {print $4}\' || echo 999999)"',
+			'  if [ -n "$available_mb" ] && [ "$available_mb" -lt "$RUNNER_JANITOR_MIN_AVAILABLE_MEMORY_MB" ]; then',
+			'    echo "ResolveIO AI runner QA resource guard: only ${available_mb}MB memory available; need ${RUNNER_JANITOR_MIN_AVAILABLE_MEMORY_MB}MB before Angular QA." | tee -a "$ARTIFACT_DIR/runner.log"',
+			'    return 7',
+			'  fi',
+			'  if [ -n "$disk_mb" ] && [ "$disk_mb" -lt "$RUNNER_JANITOR_MIN_AVAILABLE_DISK_MB" ]; then',
+			'    echo "ResolveIO AI runner QA resource guard: only ${disk_mb}MB disk available; need ${RUNNER_JANITOR_MIN_AVAILABLE_DISK_MB}MB before Angular QA." | tee -a "$ARTIFACT_DIR/runner.log"',
+			'    return 7',
+			'  fi',
+			'  return 0',
+			'}',
+			'janitor_write_cleanup_status() {',
+			'  local status="$1"',
+			'  local killed="$2"',
+			'  cat > "$(janitor_status_path)" <<EOF',
+			'{',
+			'  "status": $(janitor_json_escape "$status"),',
+			'  "killed_processes": ${killed:-0},',
+			'  "project_root": $(janitor_json_escape "$PROJECT_ROOT"),',
+			'  "updated_at": $(janitor_json_escape "$(date -u +%Y-%m-%dT%H:%M:%SZ)")',
+			'}',
+			'EOF',
+			'}',
+			''
+		].join('\n');
+	}
+}
+
+export function buildRunnerProcessJanitorShellLibrary(options: RunnerProcessJanitorScriptOptions = {}): string {
+	return RunnerProcessJanitor.buildShellLibrary(options);
+}