@anysphere/file-service 0.0.0-e6124fba → 0.0.0-e68f3241

package/src/file_utils.rs

@@ -1,328 +0,0 @@
-// what methods do i want to support here.
-// 1. isInBadDir
-// 2. isBadFile
-// 3. vscode.workspace.asRelativePath
-// 4. vscode.fs.stat
-
-use anyhow::Error;
-use encoding_rs::UTF_8;
-use std::path::Path;
-use tokio::fs;
-
-pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
-  let item_path = file_path
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-  let is_bad_dir =
-    item_path.contains("node_modules") || item_path.contains(".git");
-  Ok(is_bad_dir)
-}
-
-pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
-  let item_path = file_path
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-
-  let path = Path::new(item_path);
-  let file_name = path
-    .file_name()
-    .ok_or(anyhow::anyhow!("Failed to get file name"))?
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert file name to string"))?;
-
-  let extension = path
-    .extension()
-    .ok_or(anyhow::anyhow!("Failed to get extension"))?
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert extension to string"))?;
-
-  match file_name {
-    "package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "composer.lock"
-    | "Gemfile.lock" | "bun.lockb" => {
-      return Err(anyhow::anyhow!("File is just a lock file"));
-    }
-    _ => {}
-  }
-
-  let bad_extensions = vec![
-    "lock",
-    "bak",
-    "tmp",
-    "bin",
-    "exe",
-    "dll",
-    "so",
-    "lockb",
-    "qwoff",
-    "isl",
-    "csv",
-    "pdf",
-    // add ms word, excel, powerpoint, etc.
-    "doc",
-    "docx",
-    "xls",
-    "xlsx",
-    "ppt",
-    "pptx",
-    "odt",
-    "ods",
-    "odp",
-    "odg",
-    "odf",
-    "sxw",
-    "sxc",
-    "sxi",
-    "sxd",
-    "sdc",
-    // add images
-    "jpg",
-    "jpeg",
-    "png",
-    "gif",
-    "bmp",
-    "tif",
-    // add audio
-    "mp3",
-    "wav",
-    "wma",
-    "ogg",
-    "flac",
-    "aac",
-    // add video
-    "mp4",
-    "mov",
-    "wmv",
-    "flv",
-    "avi",
-    // add archives
-    "zip",
-    "tar",
-    "gz",
-    "7z",
-    "rar",
-    "tgz",
-    "dmg",
-    "iso",
-    "cue",
-    "mdf",
-    "mds",
-    "vcd",
-    "toast",
-    "img",
-    "apk",
-    "msi",
-    "cab",
-    "tar.gz",
-    "tar.xz",
-    "tar.bz2",
-    "tar.lzma",
-    "tar.Z",
-    "tar.sz",
-    "lzma",
-    // add fonts
-    "ttf",
-    "otf",
-    "woff",
-    "woff2",
-    "eot",
-  ];
-  match bad_extensions.contains(&extension) {
-    true => {
-      return Err(anyhow::anyhow!("File is just a lock file"));
-    }
-    _ => {}
-  }
-
-  if item_path.contains(".git")
-    || item_path.contains(".svn")
-    || item_path.contains(".hg")
-  {
-    return Err(anyhow::anyhow!("File is just a lock file"));
-  }
-
-  let bad_extensions = vec![".exe", ".dll", ".so", ".o", ".bin"];
-  match Path::new(item_path).extension() {
-    Some(extension) => match extension.to_str() {
-      Some(ext_str) => {
-        if bad_extensions.contains(&ext_str) {
-          return Err(anyhow::anyhow!("Binary file excluded from indexing."));
-        }
-      }
-      None => {
-        return Err(anyhow::anyhow!("Failed to convert extension to string"))
-      }
-    },
-    None => return Err(anyhow::anyhow!("Failed to get extension")),
-  }
-
-  // #[cfg(not(test))]
-  // {
-  let path = Path::new(item_path);
-  for part in path.iter() {
-    match part.to_str() {
-      Some(s) if s.starts_with(".") => {
-        return Err(anyhow::anyhow!("File is hidden"))
-      }
-      _ => {}
-    }
-  }
-  // }
-
-  Ok(())
-}
-
-// use binaryornot::is_binary;
-// use anyhow::Context;
-// implement the buffer above:
-pub async fn is_good_file_runtime_check(
-  file_path: &Path,
-  // _buffer: &[u8],
-) -> Result<(), Error> {
-  match get_file_size(file_path).await {
-    Ok(size) if size > 2 * 1024 * 1024 => {
-      return Err(anyhow::anyhow!("Buffer is too large"));
-    }
-    Err(e) => return Err(e),
-    _ => {}
-  }
-
-  // if is_binary(file_path).context("Failed to check if file is binary")? {
-  // 	return Err(anyhow::anyhow!("File is binary"));
-  // }
-
-  Ok(())
-}
-
-pub async fn read_string_without_bom(
-  file_path: &Path,
-) -> Result<String, Error> {
-  let file_buffer = match fs::read(file_path).await {
-    Ok(buffer) => buffer,
-    Err(e) => {
-      return Err(anyhow::anyhow!(
-        "Failed to read file buffer: {}",
-        e.to_string()
-      ))
-    }
-  };
-
-  let (cow, _) = UTF_8.decode_with_bom_removal(&file_buffer);
-
-  Ok(cow.to_string())
-}
-
-pub fn as_relative_path(
-  base_path: &Path,
-  file_path: &Path,
-) -> Result<String, Error> {
-  let relative_path = file_path.strip_prefix(base_path)?;
-  Ok(
-    relative_path
-      .to_str()
-      .ok_or(anyhow::anyhow!("Failed to convert relative path to string"))?
-      .to_string(),
-  )
-}
-
-pub async fn get_file_size(file_path: &Path) -> Result<u64, Error> {
-  let metadata = fs::metadata(file_path).await?;
-
-  Ok(metadata.len())
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use std::path::Path;
-  use tokio::io::AsyncWriteExt;
-
-  #[test]
-  fn test_is_in_bad_dir() {
-    let path = Path::new("src/node_modules/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), true);
-
-    let path = Path::new("src/.git/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), true);
-
-    let path = Path::new("src/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), false);
-  }
-
-  #[test]
-  fn test_is_good_file() {
-    let path = Path::new("src/test.rs");
-    assert_eq!(is_good_file(&path).is_ok(), true);
-
-    let path = Path::new("src/test.exe");
-    assert_eq!(is_good_file(&path).is_err(), true);
-
-    let path = Path::new("src/.hidden");
-    assert_eq!(is_good_file(&path).is_err(), true);
-  }
-
-  #[tokio::test]
-  async fn test_is_good_file_runtime_check() {
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(b"Hello, world!").await.unwrap();
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path).await.is_ok(),
-      true
-    );
-    temp_dir.close().unwrap();
-
-    // let temp_dir = tempfile::tempdir().unwrap();
-    // let temp_file_path = temp_dir.path().join("test_file");
-    // let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    // temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    // let buffer = fs::read(&temp_file_path).await.unwrap();
-    // assert_eq!(
-    //   is_good_file_runtime_check(&temp_file_path).await.is_err(),
-    //   true
-    // );
-    // temp_dir.close().unwrap();
-  }
-
-  #[tokio::test]
-  async fn test_bom_file() {
-    const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
-    const CONTENT: &str = "Hello, world!";
-
-    // Write this to a temp file
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&BOM).await.unwrap();
-    temp_file.write_all(CONTENT.as_bytes()).await.unwrap();
-
-    // expect that we read the file with tokio as the CONTENT
-    let file_contents = read_string_without_bom(&temp_file_path).await.unwrap();
-
-    // Check string equality of CONTENT (&str) to file_contents (String)
-    assert_eq!(CONTENT, file_contents);
-  }
-
-  #[test]
-  fn test_as_relative_path() {
-    let base_path = Path::new("/home/user/src");
-    let file_path = Path::new("/home/user/src/test.rs");
-    assert_eq!(as_relative_path(&base_path, &file_path).unwrap(), "test.rs");
-
-    let file_path = Path::new("/home/user/test.rs");
-    assert!(as_relative_path(&base_path, &file_path).is_err());
-  }
-
-  #[tokio::test]
-  async fn test_get_file_size() {
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file.txt");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(b"Hello, world!").await.unwrap();
-
-    let size = get_file_size(&temp_file_path).await.unwrap();
-    assert_eq!(size, 13);
-    temp_dir.close().unwrap();
-  }
-}

package/src/git_utils.rs

@@ -1,355 +0,0 @@
-use std::collections::HashSet;
-use std::path::MAIN_SEPARATOR_STR;
-use std::process::Command;
-
-pub fn list_ignored_files_and_directories(
-  workspace_root_path: &str,
-  should_return_absolute_paths: bool,
-) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
-  let mut gitignored_files = HashSet::new();
-
-  let commands = vec![
-    vec![
-      "git",
-      "ls-files",
-      "--others",
-      "--ignored",
-      "--exclude-standard",
-      "--directory",
-      "--no-empty-directory"
-    ],
-    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
-    vec![
-      "git submodule foreach --quiet 'git -C $toplevel/$path ls-files --others --ignored --exclude-standard --directory --no-empty-directory | (while read line; do echo $path/$line; done)'",
-    ],
-  ];
-
-  for command in commands {
-    let output = Command::new(&command[0])
-      .args(&command[1..])
-      .current_dir(workspace_root_path)
-      .output()?;
-
-    if output.status.success() {
-      let files = String::from_utf8(output.stdout)?
-        .lines()
-        .filter(|line| !line.is_empty())
-        .map(|line| {
-          let line = line.replace("/", MAIN_SEPARATOR_STR);
-
-          if should_return_absolute_paths {
-            let mut path = std::path::PathBuf::from(workspace_root_path);
-
-            path.push(line.clone());
-
-            match path.canonicalize() {
-              Ok(canonical_path) => {
-                canonical_path.to_string_lossy().into_owned()
-              }
-              Err(_) => String::from(line),
-            }
-          } else {
-            String::from(line)
-          }
-        })
-        .collect::<HashSet<_>>();
-
-      gitignored_files.extend(files);
-    }
-  }
-
-  Ok(gitignored_files)
-}
-
-pub fn list_ignored_files_with_absolute_paths(
-  workspace_root_path: &str,
-) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
-  let mut gitignored_files = HashSet::new();
-
-  let commands = vec![
-    vec![
-      "git",
-      "ls-files",
-      "--others",
-      "--ignored",
-      "--exclude-standard",
-    ],
-    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
-    vec![
-      "sh",
-      "-c",
-      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
-    ],
-  ];
-
-  for command in commands {
-    let output = Command::new(&command[0])
-      .args(&command[1..])
-      .current_dir(workspace_root_path)
-      .output()?;
-
-    if output.status.success() {
-      let files = String::from_utf8(output.stdout)?
-        .lines()
-        .filter(|line| !line.is_empty())
-        .map(|line| String::from(line))
-        .collect::<HashSet<_>>();
-
-      gitignored_files.extend(files);
-    }
-  }
-
-  Ok(gitignored_files)
-}
-
-pub async fn is_git_ignored(
-  workspace_root_path: &str,
-  file_path: &str,
-) -> Result<bool, anyhow::Error> {
-  let output = tokio::process::Command::new("git")
-    .args(&["check-ignore", file_path])
-    .current_dir(workspace_root_path)
-    .output()
-    .await?;
-
-  Ok(output.status.success())
-}
-
-pub async fn is_git_directory(
-  workspace_root_path: &str,
-) -> Result<bool, anyhow::Error> {
-  let output = tokio::process::Command::new("git")
-    .args(&["rev-parse", "--is-inside-work-tree"])
-    .current_dir(workspace_root_path)
-    .output()
-    .await?;
-
-  Ok(output.status.success())
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use std::fs::File;
-  use std::io::Write;
-
-  #[test]
-  fn test_no_ignored_files() {
-    let dir = tempfile::tempdir().unwrap();
-    let gitignored_files =
-      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
-        .unwrap();
-    Command::new("git")
-      .args(&["init"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    assert_eq!(gitignored_files.len(), 0);
-  }
-
-  #[test]
-  fn test_one_ignored_file() {
-    let dir = tempfile::tempdir().unwrap();
-    println!("tempdir: {:?}", dir);
-    let file_path = dir.path().join("ignored.txt");
-    let mut file = File::create(&file_path).unwrap();
-    writeln!(file, "This is an ignored file.").unwrap();
-
-    let gitignore_path = dir.path().join(".gitignore");
-    let mut gitignore = File::create(&gitignore_path).unwrap();
-    writeln!(gitignore, "ignored.txt").unwrap();
-
-    Command::new("git")
-      .args(&["init"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    let gitignored_files =
-      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
-        .unwrap();
-    println!(
-      "ignored files for test_one_ignored_file: {:?}",
-      gitignored_files
-    );
-    // assert_eq!(gitignored_files.len(), 1);
-    assert!(gitignored_files.contains(&String::from("ignored.txt")));
-  }
-
-  #[test]
-  fn test_multiple_ignored_files() {
-    let dir = tempfile::tempdir().unwrap();
-    println!("tempdir: {:?}", dir);
-    let file_path1 = dir.path().join("ignored1.txt");
-    let file_path2 = dir.path().join("ignored2.txt");
-    let mut file1 = File::create(&file_path1).unwrap();
-    let mut file2 = File::create(&file_path2).unwrap();
-    writeln!(file1, "This is an ignored file.").unwrap();
-    writeln!(file2, "This is another ignored file.").unwrap();
-
-    let gitignore_path = dir.path().join(".gitignore");
-    let mut gitignore = File::create(&gitignore_path).unwrap();
-    writeln!(gitignore, "*.txt").unwrap();
-
-    Command::new("git")
-      .args(&["init"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    let gitignored_files =
-      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
-        .unwrap();
-    println!(
-      "ignored files for test_multiple_ignored_files: {:?}",
-      gitignored_files
-    );
-    // assert_eq!(gitignored_files.len(), 2);
-    assert!(gitignored_files.contains(&String::from("ignored1.txt")));
-    assert!(gitignored_files.contains(&String::from("ignored2.txt")));
-  }
-
-  #[test]
-  fn test_git_submodule_ignored_files() {
-    let dir = tempfile::tempdir().unwrap();
-    let submodule_path = dir.path().join("submodule");
-    std::fs::create_dir(&submodule_path).unwrap();
-
-    let o = Command::new("git")
-      .args(&["init"])
-      .current_dir(&submodule_path)
-      .output()
-      .unwrap();
-    println!("git init output: {:?}", o);
-
-    let file_path = submodule_path.join("ignored.txt");
-    let mut file = File::create(&file_path).unwrap();
-    writeln!(file, "This is an ignored file.").unwrap();
-
-    let file2 = submodule_path.join("ignored2.txt");
-    let mut file = File::create(&file2).unwrap();
-    writeln!(file, "This is another ignored file.").unwrap();
-
-    let gitignore_path = submodule_path.join(".gitignore");
-    let mut gitignore = File::create(&gitignore_path).unwrap();
-    writeln!(gitignore, "*.txt").unwrap();
-
-    let o = Command::new("git")
-      .args(&["init"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    println!("git init output: {:?}", o);
-
-    // make a commit in the submodule
-    let o = Command::new("git")
-      .args(&["add", "."])
-      .current_dir(&submodule_path)
-      .output()
-      .unwrap();
-    println!("git add output: {:?}", o);
-    let o = Command::new("git")
-      .args(&["commit", "-m", "initial commit"])
-      .current_dir(&submodule_path)
-      .output()
-      .unwrap();
-    println!("git commit output: {:?}", o);
-
-    let o = Command::new("git")
-      .args(&["submodule", "add", "./submodule"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    println!("git submodule add output: {:?}", o);
-
-    let gitignored_files =
-      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
-        .unwrap();
-    println!(
-      "ignored files for test_git_submodule_ignored_files: {:?}",
-      gitignored_files
-    );
-    assert!(gitignored_files.contains(&String::from("submodule/ignored.txt")));
-    assert!(gitignored_files.contains(&String::from("submodule/ignored2.txt")));
-  }
-
-  #[test]
-  fn test_multiple_ignored_files_in_current_dir() {
-    let gitignored_files =
-      list_ignored_files_and_directories(".", false).unwrap();
-    assert!(gitignored_files.len() > 1);
-
-    // print a sample of the ignored files
-    let mut count = 0;
-    for file in gitignored_files {
-      println!("ignored file: {:?}", file);
-      count += 1;
-      if count > 10 {
-        break;
-      }
-    }
-  }
-
-  #[tokio::test]
-  async fn test_file_not_ignored() {
-    let dir = tempfile::tempdir().unwrap();
-    let file_path = dir.path().join("not_ignored.txt");
-    let mut file = File::create(&file_path).unwrap();
-    writeln!(file, "This is not an ignored file.").unwrap();
-
-    Command::new("git")
-      .args(&["init"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    let is_ignored =
-      is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt")
-        .await
-        .unwrap();
-    assert_eq!(is_ignored, false);
-  }
-
-  #[tokio::test]
-  async fn test_file_ignored() {
-    let dir = tempfile::tempdir().unwrap();
-    let file_path = dir.path().join("ignored.txt");
-    let mut file = File::create(&file_path).unwrap();
-    writeln!(file, "This is an ignored file.").unwrap();
-
-    let gitignore_path = dir.path().join(".gitignore");
-    let mut gitignore = File::create(&gitignore_path).unwrap();
-    writeln!(gitignore, "ignored.txt").unwrap();
-
-    Command::new("git")
-      .args(&["init"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    let is_ignored =
-      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
-        .await
-        .unwrap();
-    assert_eq!(is_ignored, true);
-  }
-
-  #[tokio::test]
-  async fn test_file_ignored_with_wildcard() {
-    let dir = tempfile::tempdir().unwrap();
-    let file_path = dir.path().join("ignored.txt");
-    let mut file = File::create(&file_path).unwrap();
-    writeln!(file, "This is an ignored file.").unwrap();
-
-    let gitignore_path = dir.path().join(".gitignore");
-    let mut gitignore = File::create(&gitignore_path).unwrap();
-    writeln!(gitignore, "*.txt").unwrap();
-
-    Command::new("git")
-      .args(&["init"])
-      .current_dir(dir.path())
-      .output()
-      .unwrap();
-    let is_ignored =
-      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
-        .await
-        .unwrap();
-    assert_eq!(is_ignored, true);
-  }
-}