@anysphere/file-service 0.0.0-a74cc2fc → 0.0.0-a75b800f

package/index.d.ts

@@ -3,18 +3,89 @@
 
 /* auto-generated by NAPI-RS */
 
-export class MerkleClient {
+export declare function hashBytes(bytes: Uint8Array): string
+export const enum DiffType {
+  Insert = 'Insert',
+  Delete = 'Delete',
+  Equal = 'Equal'
+}
+export interface DiffChunk {
+  diffType: DiffType
+  text: string
+}
+export interface WalkDirConfig {
+  maxNumFiles: number
+}
+export interface CommitData {
+  sha: string
+  date: number
+  message: string
+  author: string
+  parents: Array<string>
+  files: Array<CommitFile>
+}
+export interface CommitFile {
+  from: string
+  to: string
+  additions: number
+  deletions: number
+  status: CommitFileStatus
+}
+export const enum CommitFileStatus {
+  Added = 0,
+  Deleted = 1,
+  Modified = 2,
+  Renamed = 3
+}
+export const enum CommitChainGetFiles {
+  DoGetFiles = 0,
+  DontGetFiles = 1
+}
+export interface VerifyData {
+  commitTime: number
+  fileContent: string
+  filePath: string
+}
+export interface Candidate {
+  path: string
+  locations: Array<number>
+  weight: number
+}
+export declare class DiffClient {
+  constructor()
+  diff(text1: string, text2: string): Array<DiffChunk>
+  /**
+   * use https://docs.rs/diffmatchpatch/latest/diffmatchpatch/struct.DiffMatchPatch.html#method.diff_lines_to_chars
+   * then diff the chars.
+   * then convert back to lines.
+   *
+   * takes in two strings. splits based on newlines.
+   * returns diffs based on lines.
+   */
+  diffLines(text1: string, text2: string): Array<DiffChunk>
+}
+export declare class MerkleClient {
   constructor(absoluteRootDirectory: string)
-  init(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
-  computeMerkleTree(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
-  updateFile(filePath: string): Promise<void>
-  deleteFile(filePath: string): Promise<void>
+  initWithRipgrepIgnore(logFilesPath: string | undefined | null, config: WalkDirConfig): Promise<void>
   getSubtreeHash(relativePath: string): Promise<string>
   getNumEmbeddableFiles(): Promise<number>
+  getImportantPaths(k: number): Promise<Array<string>>
   getAllFiles(): Promise<Array<string>>
   getAllDirFilesToEmbed(absoluteFilePath: string): Promise<Array<string>>
-  getNextFileToEmbed(): Promise<Array<string>>
-  getSpline(absoluteFilePath: string): Promise<Array<string>>
-  getHashesForFiles(files: Array<string>): Promise<Array<string>>
-  updateRootDirectory(rootDirectory: string): void
+}
+export declare class GitClient {
+  constructor(absoluteRootDirectory: string)
+  getTotalCommitCount(): Promise<number>
+  getCommitVerifyData(commit: string): Promise<VerifyData>
+  throwIfCommitDoesntExist(rootSha: string): Promise<void>
+  getVerifyCommit(): Promise<string>
+  getRepoHeadSha(): Promise<string | null>
+  getCommitChain(hash: string, depth: number, getFiles: CommitChainGetFiles): Promise<Array<CommitData>>
+}
+export declare class GitFile {
+  findSimilarFiles(lineno: number): Promise<Array<Candidate>>
+}
+export declare class LocalGitGraph {
+  constructor(repo: string)
+  openFile(path: string): Promise<GitFile>
 }

package/index.js

@@ -224,14 +224,72 @@ switch (platform) {
         }
         break
       case 'arm':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-arm-musleabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-arm-musleabihf.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-arm-musleabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-arm-gnueabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-arm-gnueabihf.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-arm-gnueabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'riscv64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-riscv64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-riscv64-musl.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-riscv64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-riscv64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-riscv64-gnu.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-riscv64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 's390x':
         localFileExisted = existsSync(
-          join(__dirname, 'file_service.linux-arm-gnueabihf.node')
+          join(__dirname, 'file_service.linux-s390x-gnu.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./file_service.linux-arm-gnueabihf.node')
+            nativeBinding = require('./file_service.linux-s390x-gnu.node')
           } else {
-            nativeBinding = require('@anysphere/file-service-linux-arm-gnueabihf')
+            nativeBinding = require('@anysphere/file-service-linux-s390x-gnu')
           }
         } catch (e) {
           loadError = e
@@ -252,6 +310,14 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { MerkleClient } = nativeBinding
+const { hashBytes, DiffType, DiffClient, MerkleClient, GitClient, CommitFileStatus, CommitChainGetFiles, GitFile, LocalGitGraph } = nativeBinding
 
+module.exports.hashBytes = hashBytes
+module.exports.DiffType = DiffType
+module.exports.DiffClient = DiffClient
 module.exports.MerkleClient = MerkleClient
+module.exports.GitClient = GitClient
+module.exports.CommitFileStatus = CommitFileStatus
+module.exports.CommitChainGetFiles = CommitChainGetFiles
+module.exports.GitFile = GitFile
+module.exports.LocalGitGraph = LocalGitGraph

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-a74cc2fc",
+  "version": "0.0.0-a75b800f",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -14,9 +14,13 @@
       ]
     }
   },
+  "files": [
+    "index.js",
+    "index.d.ts"
+  ],
   "license": "MIT",
   "devDependencies": {
-    "@napi-rs/cli": "^2.16.2",
+    "@napi-rs/cli": "^2.18.4",
     "ava": "^5.1.1"
   },
   "ava": {
@@ -36,12 +40,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-a74cc2fc",
-    "@anysphere/file-service-darwin-x64": "0.0.0-a74cc2fc",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-a74cc2fc",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-a74cc2fc",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-a74cc2fc",
-    "@anysphere/file-service-darwin-universal": "0.0.0-a74cc2fc",
-    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-a74cc2fc"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-a75b800f",
+    "@anysphere/file-service-darwin-x64": "0.0.0-a75b800f",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-a75b800f",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-a75b800f",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-a75b800f",
+    "@anysphere/file-service-darwin-universal": "0.0.0-a75b800f",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-a75b800f"
   }
 }

package/.yarnrc.yml

@@ -1 +0,0 @@
-nodeLinker: node-modules

package/Cargo.toml

@@ -1,43 +0,0 @@
-[package]
-edition = "2021"
-name = "file_service"
-version = "0.0.0"
-
-[lib]
-crate-type = ["cdylib"]
-
-[features]
-default = ["windows-subsystem"]
-windows-subsystem = []
-debugfile = []
-
-[dependencies]
-# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2.12.2", default-features = false, features = ["napi4", "async", "tokio_rt"] }
-napi-derive = "2.12.2"
-tokio = { version = "1.32.0", features = ["process", "full"] }
-sha2 = "0.10.7"
-rand = "0.8.5"
-tempfile = "3.8.0"
-anyhow = "1.0.75"
-tonic = "0.9.2"
-prost = "0.11.9"
-tracing = "0.1.37"
-tracing-subscriber = "0.3.17"
-tracing-appender = "0.2.2"
-binaryornot = "1.0.0"
-dunce = "1.0.1"
-encoding_rs = "0.8.33"
-
-[target.'cfg(not(target_os = "linux"))'.dependencies]
-tracing-axiom = "0.4"
-
-[build-dependencies]
-napi-build = "2.0.1"
-tonic-build = "0.9.2"
-anyhow = "1.0.75"
-glob = "0.3.0"
-
-
-[profile.release]
-lto = true

package/build.rs

@@ -1,46 +0,0 @@
-use std::path::Path;
-
-extern crate napi_build;
-
-fn main() -> Result<(), anyhow::Error> {
-  #[cfg(target_os = "windows")]
-  println!("cargo:rustc-cdylib-link-arg=/SUBSYSTEM:WINDOWS");
-  napi_build::setup();
-
-  // print the relative path.
-  // let workspace_root = "../../../../../";
-  // let path = std::path::Path::new(workspace_root).canonicalize()?;
-  // let include_path = Path::join(&path, "schema");
-
-  // // let relevant_protos = &[
-	// // 	"aiserver/v1/repository.proto",
-	// // 	"aiserver/v1/symbolic_context.proto",
-	// // 	"aiserver/v1/utils.proto"
-	// // ];
-  // // let proto_paths = relevant_protos
-  // //   .iter()
-  // //   .map(|proto| Path::join(&include_path, proto))
-	// // 	.collect::<Vec<_>>();
-	// let proto_glob = Path::join(&include_path, "aiserver/v1/*.proto");
-  // let relevant_protos: Vec<_> = glob::glob(proto_glob.to_str().expect("Failed to convert path to str"))?
-  //   .filter_map(Result::ok)
-  //   .collect();
-
-  // let proto_paths = relevant_protos
-  //   .iter()
-  //   .map(|proto_path| proto_path.to_str().expect("Failed to convert path to str"))
-  //   .collect::<Vec<_>>();
-  // let includes = &[include_path.to_str().unwrap()];
-
-  // // print the path
-  // println!("cargo:rustc-env=INCLUDE_PATH={}", include_path.display());
-
-  // tonic_build::configure()
-  //   .build_server(false)
-	// 	.build_transport(true)
-	// 	.out_dir("src/proto")
-  //   .compile(&proto_paths, includes)?;
-
-
-	Ok(())
-}

package/src/file_utils.rs

@@ -1,328 +0,0 @@
-// what methods do i want to support here.
-// 1. isInBadDir
-// 2. isBadFile
-// 3. vscode.workspace.asRelativePath
-// 4. vscode.fs.stat
-
-use anyhow::Error;
-use encoding_rs::UTF_8;
-use std::path::Path;
-use tokio::fs;
-
-pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
-  let item_path = file_path
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-  let is_bad_dir =
-    item_path.contains("node_modules") || item_path.contains(".git");
-  Ok(is_bad_dir)
-}
-
-pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
-  let item_path = file_path
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-
-  let path = Path::new(item_path);
-  let file_name = path
-    .file_name()
-    .ok_or(anyhow::anyhow!("Failed to get file name"))?
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert file name to string"))?;
-
-  let extension = path
-    .extension()
-    .ok_or(anyhow::anyhow!("Failed to get extension"))?
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert extension to string"))?;
-
-  match file_name {
-    "package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "composer.lock"
-    | "Gemfile.lock" | "bun.lockb" => {
-      return Err(anyhow::anyhow!("File is just a lock file"));
-    }
-    _ => {}
-  }
-
-  let bad_extensions = vec![
-    "lock",
-    "bak",
-    "tmp",
-    "bin",
-    "exe",
-    "dll",
-    "so",
-    "lockb",
-    "qwoff",
-    "isl",
-    "csv",
-    "pdf",
-    // add ms word, excel, powerpoint, etc.
-    "doc",
-    "docx",
-    "xls",
-    "xlsx",
-    "ppt",
-    "pptx",
-    "odt",
-    "ods",
-    "odp",
-    "odg",
-    "odf",
-    "sxw",
-    "sxc",
-    "sxi",
-    "sxd",
-    "sdc",
-    // add images
-    "jpg",
-    "jpeg",
-    "png",
-    "gif",
-    "bmp",
-    "tif",
-    // add audio
-    "mp3",
-    "wav",
-    "wma",
-    "ogg",
-    "flac",
-    "aac",
-    // add video
-    "mp4",
-    "mov",
-    "wmv",
-    "flv",
-    "avi",
-    // add archives
-    "zip",
-    "tar",
-    "gz",
-    "7z",
-    "rar",
-    "tgz",
-    "dmg",
-    "iso",
-    "cue",
-    "mdf",
-    "mds",
-    "vcd",
-    "toast",
-    "img",
-    "apk",
-    "msi",
-    "cab",
-    "tar.gz",
-    "tar.xz",
-    "tar.bz2",
-    "tar.lzma",
-    "tar.Z",
-    "tar.sz",
-    "lzma",
-    // add fonts
-    "ttf",
-    "otf",
-    "woff",
-    "woff2",
-    "eot",
-  ];
-  match bad_extensions.contains(&extension) {
-    true => {
-      return Err(anyhow::anyhow!("File is just a lock file"));
-    }
-    _ => {}
-  }
-
-  if item_path.contains(".git")
-    || item_path.contains(".svn")
-    || item_path.contains(".hg")
-  {
-    return Err(anyhow::anyhow!("File is just a lock file"));
-  }
-
-  let bad_extensions = vec![".exe", ".dll", ".so", ".o", ".bin"];
-  match Path::new(item_path).extension() {
-    Some(extension) => match extension.to_str() {
-      Some(ext_str) => {
-        if bad_extensions.contains(&ext_str) {
-          return Err(anyhow::anyhow!("Binary file excluded from indexing."));
-        }
-      }
-      None => {
-        return Err(anyhow::anyhow!("Failed to convert extension to string"))
-      }
-    },
-    None => return Err(anyhow::anyhow!("Failed to get extension")),
-  }
-
-  // #[cfg(not(test))]
-  // {
-  let path = Path::new(item_path);
-  for part in path.iter() {
-    match part.to_str() {
-      Some(s) if s.starts_with(".") => {
-        return Err(anyhow::anyhow!("File is hidden"))
-      }
-      _ => {}
-    }
-  }
-  // }
-
-  Ok(())
-}
-
-// use binaryornot::is_binary;
-// use anyhow::Context;
-// implement the buffer above:
-pub async fn is_good_file_runtime_check(
-  file_path: &Path,
-  // _buffer: &[u8],
-) -> Result<(), Error> {
-  match get_file_size(file_path).await {
-    Ok(size) if size > 2 * 1024 * 1024 => {
-      return Err(anyhow::anyhow!("Buffer is too large"));
-    }
-    Err(e) => return Err(e),
-    _ => {}
-  }
-
-  // if is_binary(file_path).context("Failed to check if file is binary")? {
-  // 	return Err(anyhow::anyhow!("File is binary"));
-  // }
-
-  Ok(())
-}
-
-pub async fn read_string_without_bom(
-  file_path: &Path,
-) -> Result<String, Error> {
-  let file_buffer = match fs::read(file_path).await {
-    Ok(buffer) => buffer,
-    Err(e) => {
-      return Err(anyhow::anyhow!(
-        "Failed to read file buffer: {}",
-        e.to_string()
-      ))
-    }
-  };
-
-  let (cow, _) = UTF_8.decode_with_bom_removal(&file_buffer);
-
-  Ok(cow.to_string())
-}
-
-pub fn as_relative_path(
-  base_path: &Path,
-  file_path: &Path,
-) -> Result<String, Error> {
-  let relative_path = file_path.strip_prefix(base_path)?;
-  Ok(
-    relative_path
-      .to_str()
-      .ok_or(anyhow::anyhow!("Failed to convert relative path to string"))?
-      .to_string(),
-  )
-}
-
-pub async fn get_file_size(file_path: &Path) -> Result<u64, Error> {
-  let metadata = fs::metadata(file_path).await?;
-
-  Ok(metadata.len())
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use std::path::Path;
-  use tokio::io::AsyncWriteExt;
-
-  #[test]
-  fn test_is_in_bad_dir() {
-    let path = Path::new("src/node_modules/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), true);
-
-    let path = Path::new("src/.git/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), true);
-
-    let path = Path::new("src/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), false);
-  }
-
-  #[test]
-  fn test_is_good_file() {
-    let path = Path::new("src/test.rs");
-    assert_eq!(is_good_file(&path).is_ok(), true);
-
-    let path = Path::new("src/test.exe");
-    assert_eq!(is_good_file(&path).is_err(), true);
-
-    let path = Path::new("src/.hidden");
-    assert_eq!(is_good_file(&path).is_err(), true);
-  }
-
-  #[tokio::test]
-  async fn test_is_good_file_runtime_check() {
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(b"Hello, world!").await.unwrap();
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path).await.is_ok(),
-      true
-    );
-    temp_dir.close().unwrap();
-
-    // let temp_dir = tempfile::tempdir().unwrap();
-    // let temp_file_path = temp_dir.path().join("test_file");
-    // let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    // temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    // let buffer = fs::read(&temp_file_path).await.unwrap();
-    // assert_eq!(
-    //   is_good_file_runtime_check(&temp_file_path).await.is_err(),
-    //   true
-    // );
-    // temp_dir.close().unwrap();
-  }
-
-  #[tokio::test]
-  async fn test_bom_file() {
-    const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
-    const CONTENT: &str = "Hello, world!";
-
-    // Write this to a temp file
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&BOM).await.unwrap();
-    temp_file.write_all(CONTENT.as_bytes()).await.unwrap();
-
-    // expect that we read the file with tokio as the CONTENT
-    let file_contents = read_string_without_bom(&temp_file_path).await.unwrap();
-
-    // Check string equality of CONTENT (&str) to file_contents (String)
-    assert_eq!(CONTENT, file_contents);
-  }
-
-  #[test]
-  fn test_as_relative_path() {
-    let base_path = Path::new("/home/user/src");
-    let file_path = Path::new("/home/user/src/test.rs");
-    assert_eq!(as_relative_path(&base_path, &file_path).unwrap(), "test.rs");
-
-    let file_path = Path::new("/home/user/test.rs");
-    assert!(as_relative_path(&base_path, &file_path).is_err());
-  }
-
-  #[tokio::test]
-  async fn test_get_file_size() {
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file.txt");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(b"Hello, world!").await.unwrap();
-
-    let size = get_file_size(&temp_file_path).await.unwrap();
-    assert_eq!(size, 13);
-    temp_dir.close().unwrap();
-  }
-}