packagepurge 1.0.0

package/core/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "packagepurge_core"
+version = "0.1.0"
+authors = ["Eng. Onyango Benard"]
+edition = "2021"
+
+[dependencies]
+walkdir = "2.5"
+rayon = "1.10"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serde_yaml = "0.9"
+sha2 = "0.10"
+hex = "0.4"
+chrono = { version = "0.4", features = ["serde"] }
+anyhow = "1.0"
+clap = { version = "4.5", features = ["derive"] }
+fs_extra = "1.3"
+ignore = "0.4"
+regex = "1.11"
+dirs = "5.0"
+tempfile = "3.10"

package/core/src/arc_lfu.rs

@@ -0,0 +1,91 @@
+#![allow(dead_code)]
+use std::collections::{HashMap, VecDeque};
+
+// SLRU: probationary and protected segments, each LRU-like (front=MRU, back=LRU)
+pub struct SlruPolicy {
+	probationary: VecDeque<String>,
+	protected: VecDeque<String>,
+	cap_probationary: usize,
+	cap_protected: usize,
+	in_probationary: HashMap<String, bool>,
+	in_protected: HashMap<String, bool>,
+}
+
+impl SlruPolicy {
+	pub fn new(capacity: usize) -> Self {
+		let cap_protected = (capacity as f32 * 0.8) as usize;
+		let cap_probationary = capacity.saturating_sub(cap_protected);
+		Self {
+			probationary: VecDeque::new(),
+			protected: VecDeque::new(),
+			cap_probationary,
+			cap_protected,
+			in_probationary: HashMap::new(),
+			in_protected: HashMap::new(),
+		}
+	}
+
+	pub fn record_hit(&mut self, key: &str) {
+		let k = key.to_string();
+		if self.in_protected.remove(&k).is_some() {
+			self.protected.retain(|x| x != &k);
+			self.protected.push_front(k.clone());
+			self.in_protected.insert(k, true);
+			return;
+		}
+		if self.in_probationary.remove(&k).is_some() {
+			// promote to protected
+			self.probationary.retain(|x| x != &k);
+			self.protected.push_front(k.clone());
+			self.in_protected.insert(k.clone(), true);
+			// enforce protected capacity
+			while self.protected.len() > self.cap_protected {
+				if let Some(v) = self.protected.pop_back() { self.in_protected.remove(&v); }
+			}
+			return;
+		}
+		// new entry goes to probationary
+		self.probationary.push_front(k.clone());
+		self.in_probationary.insert(k.clone(), true);
+		while self.probationary.len() > self.cap_probationary {
+			if let Some(v) = self.probationary.pop_back() { self.in_probationary.remove(&v); }
+		}
+	}
+
+	pub fn select_victim(&mut self) -> Option<String> {
+		if let Some(v) = self.probationary.pop_back() { self.in_probationary.remove(&v); return Some(v); }
+		if let Some(v) = self.protected.pop_back() { self.in_protected.remove(&v); return Some(v); }
+		None
+	}
+}
+
+// Simple LFU: key->freq, and buckets freq->VecDeque keys. Evicts from lowest freq, oldest within bucket
+pub struct SimpleLfu {
+	freq: HashMap<String, usize>,
+	buckets: HashMap<usize, VecDeque<String>>,
+}
+
+impl SimpleLfu {
+	pub fn new() -> Self { Self { freq: HashMap::new(), buckets: HashMap::new() } }
+
+	pub fn increment(&mut self, key: &str) {
+		let k = key.to_string();
+		let f = *self.freq.get(&k).unwrap_or(&0);
+		if let Some(q) = self.buckets.get_mut(&f) { q.retain(|x| x != &k); }
+		let nf = f + 1;
+		self.freq.insert(k.clone(), nf);
+		self.buckets.entry(nf).or_default().push_front(k);
+	}
+
+	pub fn victim(&mut self) -> Option<String> {
+		if self.freq.is_empty() { return None; }
+		let minf = *self.freq.values().min().unwrap_or(&0);
+		if let Some(q) = self.buckets.get_mut(&minf) {
+			if let Some(k) = q.pop_back() {
+				self.freq.remove(&k);
+				return Some(k);
+			}
+		}
+		None
+	}
+}

package/core/src/cache.rs

@@ -0,0 +1,205 @@
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::rc::Rc;
+use chrono::Utc;
+use crate::types::PackageUsageMetrics;
+
+// Doubly-linked list node
+struct Node<K, V> {
+	key: K,
+	value: V,
+	prev: Option<Rc<RefCell<Node<K, V>>>>,
+	next: Option<Rc<RefCell<Node<K, V>>>>,
+}
+
+pub struct LruCache<K, V> where K: Eq + Hash + Clone {
+	capacity: usize,
+	map: HashMap<K, Rc<RefCell<Node<K, V>>>>,
+	head: Option<Rc<RefCell<Node<K, V>>>>, // MRU
+	tail: Option<Rc<RefCell<Node<K, V>>>>, // LRU
+}
+
+impl<K, V> LruCache<K, V> where K: Eq + Hash + Clone {
+	pub fn new(capacity: usize) -> Self {
+		Self { capacity, map: HashMap::new(), head: None, tail: None }
+	}
+
+	#[allow(dead_code)]
+	pub fn len(&self) -> usize { self.map.len() }
+	#[allow(dead_code)]
+	pub fn is_empty(&self) -> bool { self.map.is_empty() }
+
+	pub fn get(&mut self, key: &K) -> Option<V> where V: Clone {
+		if let Some(node_rc) = self.map.get(key).cloned() {
+			self.move_to_head(node_rc.clone());
+			return Some(node_rc.borrow().value.clone());
+		}
+		None
+	}
+
+	pub fn put(&mut self, key: K, value: V) -> Option<(K, V)> where V: Clone {
+		if let Some(node_rc) = self.map.get(&key).cloned() {
+			// Update value and move to head
+			node_rc.borrow_mut().value = value;
+			self.move_to_head(node_rc);
+			return None;
+		}
+		// Insert new
+		let node = Rc::new(RefCell::new(Node { key: key.clone(), value, prev: None, next: None }));
+		self.attach_head(node.clone());
+		self.map.insert(key.clone(), node);
+		// Evict if over capacity
+		if self.map.len() > self.capacity {
+			if let Some(lru) = self.pop_tail() {
+				let k = lru.borrow().key.clone();
+				let v = lru.borrow().value.clone();
+				self.map.remove(&k);
+				return Some((k, v));
+			}
+		}
+		None
+	}
+
+	fn detach(&mut self, node: Rc<RefCell<Node<K, V>>>) {
+		let prev = node.borrow().prev.clone();
+		let next = node.borrow().next.clone();
+		if let Some(p) = prev.clone() { p.borrow_mut().next = next.clone(); } else { self.head = next.clone(); }
+		if let Some(n) = next.clone() { n.borrow_mut().prev = prev.clone(); } else { self.tail = prev.clone(); }
+		node.borrow_mut().prev = None;
+		node.borrow_mut().next = None;
+	}
+
+	fn attach_head(&mut self, node: Rc<RefCell<Node<K, V>>>) {
+		node.borrow_mut().prev = None;
+		node.borrow_mut().next = self.head.clone();
+		if let Some(h) = self.head.clone() { h.borrow_mut().prev = Some(node.clone()); }
+		self.head = Some(node.clone());
+		if self.tail.is_none() { self.tail = Some(node); }
+	}
+
+	fn move_to_head(&mut self, node: Rc<RefCell<Node<K, V>>>) {
+		self.detach(node.clone());
+		self.attach_head(node);
+	}
+
+	fn pop_tail(&mut self) -> Option<Rc<RefCell<Node<K, V>>>> {
+		if let Some(t) = self.tail.clone() {
+			self.detach(t.clone());
+			return Some(t);
+		}
+		None
+	}
+}
+
+/// LRU cache specialized for package versions with usage tracking
+#[allow(dead_code)]
+pub struct PackageLruCache {
+	cache: LruCache<String, PackageUsageMetrics>,
+	max_size_bytes: u64,
+	current_size_bytes: u64,
+}
+
+impl PackageLruCache {
+	pub fn new(max_packages: usize, max_size_bytes: u64) -> Self {
+		Self {
+			cache: LruCache::new(max_packages),
+			max_size_bytes,
+			current_size_bytes: 0,
+		}
+	}
+
+	/// Record package access (updates atime and increments access count)
+	pub fn record_access(&mut self, package_key: &str, size_bytes: u64) {
+		let now = Utc::now();
+		if let Some(metrics) = self.cache.get(&package_key.to_string()) {
+			// Update existing metrics
+			let mut updated = metrics;
+			updated.last_access_time = now;
+			updated.access_count += 1;
+			self.cache.put(package_key.to_string(), updated);
+		} else {
+			// Create new metrics
+			let metrics = PackageUsageMetrics {
+				package_key: package_key.to_string(),
+				last_access_time: now,
+				last_script_execution: None,
+				access_count: 1,
+				script_execution_count: 0,
+				last_successful_build: None,
+			};
+			if let Some((_evicted_key, _evicted_metrics)) = self.cache.put(package_key.to_string(), metrics) {
+				// Handle eviction if needed
+				// In a full implementation, we'd track size_bytes per package
+			}
+			self.current_size_bytes += size_bytes;
+		}
+	}
+
+	/// Record successful script execution
+	#[allow(dead_code)]
+	pub fn record_script_execution(&mut self, package_key: &str) {
+		let now = Utc::now();
+		if let Some(metrics) = self.cache.get(&package_key.to_string()) {
+			let mut updated = metrics;
+			updated.last_script_execution = Some(now);
+			updated.script_execution_count += 1;
+			self.cache.put(package_key.to_string(), updated);
+		}
+	}
+
+	/// Record successful build
+	#[allow(dead_code)]
+	pub fn record_build(&mut self, package_key: &str) {
+		let now = Utc::now();
+		if let Some(metrics) = self.cache.get(&package_key.to_string()) {
+			let mut updated = metrics;
+			updated.last_successful_build = Some(now);
+			self.cache.put(package_key.to_string(), updated);
+		}
+	}
+
+	/// Get metrics for a package (updates LRU position)
+	pub fn get_metrics(&mut self, package_key: &str) -> Option<PackageUsageMetrics> {
+		self.cache.get(&package_key.to_string())
+	}
+
+	/// Get least recently used packages (for eviction candidates)
+	#[allow(dead_code)]
+	pub fn get_lru_packages(&self, _count: usize) -> Vec<String> {
+		// This is a simplified version - in a full implementation,
+		// we'd need to iterate through the tail of the LRU cache
+		Vec::new() // Placeholder
+	}
+
+	/// Check if package should be kept based on LRU strategy
+	pub fn should_keep_lru(&mut self, package_key: &str, days_threshold: i64) -> bool {
+		if let Some(metrics) = self.get_metrics(package_key) {
+			let days_since_access = (Utc::now() - metrics.last_access_time).num_days();
+			return days_since_access < days_threshold;
+		}
+		false
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::LruCache;
+	#[test]
+	fn test_lru_basic() {
+		let mut lru = LruCache::new(2);
+		assert!(lru.get(&"a").is_none());
+		assert!(lru.put("a", 1).is_none());
+		assert_eq!(lru.get(&"a"), Some(1));
+		assert!(lru.put("b", 2).is_none());
+		assert_eq!(lru.len(), 2);
+		// Insert c -> evict LRU (which should be 'a' after accessing 'a' it's MRU; LRU is 'b'?)
+		// Access changes MRU; sequence ensures eviction is correct
+		lru.get(&"a"); // 'a' MRU, 'b' LRU
+		let evicted = lru.put("c", 3);
+		assert!(evicted.is_some());
+		let (k, v) = evicted.unwrap();
+		assert_eq!(k, "b");
+		assert_eq!(v, 2);
+	}
+}

package/core/src/lockfiles.rs

@@ -0,0 +1,112 @@
+use std::fs;
+use std::path::Path;
+
+pub type DepList = Vec<(String, String)>; // (name, version)
+
+pub fn parse_npm_package_lock(path: &Path) -> DepList {
+	let mut deps_list: DepList = Vec::new();
+	let text = match fs::read_to_string(path) { Ok(t) => t, Err(_) => return deps_list };
+	let json: serde_json::Value = match serde_json::from_str(&text) { Ok(v) => v, Err(_) => return deps_list };
+	
+	fn walk(node: &serde_json::Value, list: &mut DepList) {
+		if let Some(deps) = node.get("dependencies").and_then(|d| d.as_object()) {
+			for (name, dep_node) in deps {
+				if let Some(ver) = dep_node.get("version").and_then(|v| v.as_str()) {
+					list.push((name.clone(), ver.to_string()));
+				}
+				walk(dep_node, list);
+			}
+		}
+        // Handle 'packages' in lockfile v2/v3
+        if let Some(packages) = node.get("packages").and_then(|d| d.as_object()) {
+            for (key, pkg_node) in packages {
+                if key.is_empty() { continue; } // Root
+                
+                // Key is path like "node_modules/pkg" or "node_modules/a/node_modules/b"
+                // We want the package name, which is after the last "node_modules/"
+                let name = if let Some(idx) = key.rfind("node_modules/") {
+                    key[idx + "node_modules/".len()..].to_string()
+                } else {
+                    key.clone()
+                };
+
+                if let Some(ver) = pkg_node.get("version").and_then(|v| v.as_str()) {
+                    list.push((name, ver.to_string()));
+                }
+            }
+        }
+	}
+    
+	walk(&json, &mut deps_list);
+	deps_list
+}
+
+pub fn parse_yarn_lock(path: &Path) -> DepList {
+	let mut list: DepList = Vec::new();
+	let text = match fs::read_to_string(path) { Ok(t) => t, Err(_) => return list };
+	
+	let mut current_name: Option<String> = None;
+    
+	for line in text.lines() {
+		let trimmed = line.trim();
+        if trimmed.is_empty() || trimmed.starts_with('#') { continue; }
+        
+		if !line.starts_with(' ') {
+            // Start of a block: "pkg@ver, pkg@ver:"
+            let parts: Vec<&str> = trimmed.trim_end_matches(':').split(',').collect();
+            if let Some(first) = parts.first() {
+                // Extract name from "name@range"
+                // This is heuristic; yarn lock keys are complex.
+                // Simpler: wait for "version" line.
+                // But we need the name.
+                // Pattern: name@^1.2.3
+                // Last '@' separates name and version range, but scoped packages start with @.
+                let s = first.trim().trim_matches('"');
+                if let Some(idx) = s.rfind('@') {
+                    if idx > 0 {
+                        current_name = Some(s[..idx].to_string());
+                    } else {
+                        current_name = None;
+                    }
+                }
+            }
+		} else if let Some(name) = &current_name {
+            if trimmed.starts_with("version") {
+                // version "1.2.3"
+                let parts: Vec<&str> = trimmed.split_whitespace().collect();
+                if parts.len() >= 2 {
+                    let ver = parts[1].trim_matches('"');
+                    list.push((name.clone(), ver.to_string()));
+                    current_name = None; // Reset so we don't duplicate
+                }
+            }
+        }
+	}
+	list
+}
+
+pub fn parse_pnpm_lock(path: &Path) -> DepList {
+	let mut list: DepList = Vec::new();
+	let text = match fs::read_to_string(path) { Ok(t) => t, Err(_) => return list };
+	
+	for line in text.lines() {
+		let l = line.trim();
+        // /name/version:
+		if l.starts_with('/') && l.ends_with(':') {
+            let content = l.trim_end_matches(':');
+            // content is like /@babel/core/7.2.0
+            // extract name and version.
+            // Split by '/'
+            let parts: Vec<&str> = content.split('/').collect();
+            // parts[0] is empty
+            // if scoped: "", "@scope", "pkg", "ver" -> len 4
+            // if unscoped: "", "pkg", "ver" -> len 3
+            if parts.len() >= 3 {
+                let ver = parts.last().unwrap().to_string();
+                let name = parts[1..parts.len()-1].join("/");
+                list.push((name, ver));
+            }
+		}
+	}
+	list
+}

package/core/src/main.rs

@@ -0,0 +1,125 @@
+mod types;
+mod scanner;
+mod safety;
+mod optimization;
+mod cache;
+mod ml;
+mod arc_lfu;
+mod lockfiles;
+mod symlink;
+mod usage_tracker;
+
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+use std::path::PathBuf;
+
+use optimization::{plan_basic_cleanup, RulesConfig, OptimizationEngine};
+
+#[derive(Parser)]
+#[command(name = "packagepurge-core", version)]
+struct Cli {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    /// Scan filesystem and output dependency/caches JSON
+    Scan { #[arg(short, long)] paths: Vec<PathBuf> },
+    /// Produce cleanup plan without mutating filesystem
+    DryRun { #[arg(short, long, default_value_t = 90)] preserve_days: i64, #[arg(short, long)] paths: Vec<PathBuf> },
+    /// Move targets to quarantine (atomic move) based on paths provided
+    Quarantine { #[arg(required=true)] targets: Vec<PathBuf> },
+    /// Rollback by id or latest
+    Rollback {
+        #[arg(long)] id: Option<String>,
+        #[arg(long)] latest: bool,
+    },
+    /// Optimize with ML/LRU and symlinking (dry run)
+    Optimize {
+        #[arg(short, long, default_value_t = 90)] preserve_days: i64,
+        #[arg(short, long)] paths: Vec<PathBuf>,
+        #[arg(long)] enable_symlinking: bool,
+        #[arg(long)] enable_ml: bool,
+        #[arg(long, default_value_t = 1000)] lru_max_packages: usize,
+        #[arg(long, default_value_t = 10_000_000_000)] lru_max_size_bytes: u64,
+    },
+    /// Execute symlinking for duplicate packages
+    Symlink {
+        #[arg(short, long)] paths: Vec<PathBuf>,
+    },
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    match cli.command {
+        Commands::Scan { paths } => {
+            let out = scanner::scan(&paths)?;
+            println!("{}", serde_json::to_string_pretty(&out)?);
+        }
+        Commands::DryRun { preserve_days, paths } => {
+            let scan = scanner::scan(&paths)?;
+            let report = plan_basic_cleanup(&scan, &RulesConfig {
+                preserve_days,
+                enable_symlinking: false,
+                enable_ml_prediction: false,
+                lru_max_packages: 1000,
+                lru_max_size_bytes: 10_000_000_000, // 10GB default
+            })?;
+            println!("{}", serde_json::to_string_pretty(&report)?);
+        }
+        Commands::Quarantine { targets } => {
+            let mut recs = Vec::new();
+            for t in targets {
+                match safety::move_to_quarantine(&t) {
+                    Ok(r) => recs.push(r),
+                    Err(e) => eprintln!("Failed to quarantine {:?}: {}", t, e),
+                }
+            }
+            println!("{}", serde_json::to_string_pretty(&recs)?);
+        }
+        Commands::Rollback { id, latest } => {
+            let rec = if let Some(i) = id { safety::find_quarantine_by_id(&i) } else if latest { safety::latest_quarantine() } else { None };
+            if let Some(r) = rec {
+                if let Err(e) = safety::rollback_record(&r) {
+                    eprintln!("{}", e);
+                    std::process::exit(1);
+                }
+                println!("{}", serde_json::to_string_pretty(&serde_json::json!({"status":"ok","id": r.id}))?);
+            } else {
+                eprintln!("No matching quarantine record found");
+                std::process::exit(2);
+            }
+        }
+        Commands::Optimize { preserve_days, paths, enable_symlinking, enable_ml, lru_max_packages, lru_max_size_bytes } => {
+            let scan = scanner::scan(&paths)?;
+            let config = RulesConfig {
+                preserve_days,
+                enable_symlinking,
+                enable_ml_prediction: enable_ml,
+                lru_max_packages,
+                lru_max_size_bytes,
+            };
+            let mut engine = OptimizationEngine::new(config)?;
+            let report = engine.plan_optimized_cleanup(&scan)?;
+            println!("{}", serde_json::to_string_pretty(&report)?);
+        }
+        Commands::Symlink { paths } => {
+            let scan = scanner::scan(&paths)?;
+            let config = RulesConfig {
+                preserve_days: 90,
+                enable_symlinking: true,
+                enable_ml_prediction: false,
+                lru_max_packages: 1000,
+                lru_max_size_bytes: 10_000_000_000,
+            };
+            let engine = OptimizationEngine::new(config)?;
+            let count = engine.execute_symlinking(&scan)?;
+            println!("{}", serde_json::to_string_pretty(&serde_json::json!({
+                "status": "ok",
+                "symlinked_count": count
+            }))?);
+        }
+    }
+    Ok(())
+}