@anysphere/file-service 0.0.0-eb8b99bf → 0.0.0-eb962576

package/index.d.ts

@@ -3,18 +3,189 @@
 
 /* auto-generated by NAPI-RS */
 
-export class MerkleClient {
-  constructor(rootDirectory: string)
-  init(): Promise<void>
-  computeMerkleTree(): Promise<void>
-  updateFile(filePath: string): Promise<void>
-  deleteFile(filePath: string): Promise<void>
+export const enum DiffType {
+  Insert = 'Insert',
+  Delete = 'Delete',
+  Equal = 'Equal'
+}
+export interface DiffChunk {
+  diffType: DiffType
+  text: string
+}
+export interface CommitData {
+  sha: string
+  date: number
+  message: string
+  author: string
+  parents: Array<string>
+  files: Array<CommitFile>
+}
+export interface CommitFile {
+  from: string
+  to: string
+  additions: number
+  deletions: number
+  status: CommitFileStatus
+}
+export const enum CommitFileStatus {
+  Added = 0,
+  Deleted = 1,
+  Modified = 2,
+  Renamed = 3
+}
+export const enum CommitChainGetFiles {
+  DoGetFiles = 0,
+  DontGetFiles = 1
+}
+export interface VerifyData {
+  commitTime: number
+  fileContent: string
+  filePath: string
+}
+export interface IndexedFile {
+  path: string
+  diff: Array<string>
+}
+export interface IndexedPullRequest {
+  prNumber: number
+  generation: number
+  sha: string
+  commitSecret: string
+  body: string
+  changedFiles: Array<IndexedFile>
+  unixTimestamp: number
+  trimmedTooLarge: boolean
+  title?: string
+  author?: string
+  date?: Date
+}
+export interface JsPullRequestExtractionOptions {
+  lastIndexedCommit?: string
+  lastIndexedCommitGeneration?: number
+  syncBitmap?: Buffer
+  limitCommits?: number
+  limitPerPr?: number
+  context?: number
+  workerThreads?: number
+  jsConcurrency?: number
+  jsChunkSize?: number
+}
+export interface JsPullRequestHydrationOptions {
+  limitPerPr?: number
+  context?: number
+  workerThreads?: number
+  interestingPaths?: Array<string>
+}
+export interface JsHandshakeData {
+  lastSeenCommit: string
+  lastSeenCommitSecret: string
+  origin: string
+}
+export interface JsRepositoryTip {
+  commitSha: string
+  commitShaSecret: string
+  generation: number
+}
+export interface ExtractResults {
+  repositoryTip?: JsRepositoryTip
+  progress: IndexedPullRequestProgress
+  interrupted: boolean
+}
+export interface IndexedPullRequestProgress {
+  /** Total number of commits in the repository */
+  total: number
+  /** Number of commits that have been synced to the server */
+  synced: number
+  /** Number of commits that have been synced to the server in this run */
+  newSynced: number
+  /** Number of commits that have been marked as ignored in this run */
+  newIgnored: number
+  /** Number of commits that have failed to be indexed in this run */
+  newFailed: number
+  /** Time (in seconds) since the start of the indexing process */
+  elapsed: number
+}
+export const MULTI_ROOT_ABSOLUTE_PATH: string
+export interface WalkDirConfig {
+  maxNumFiles: number
+  logFilePath?: string
+  blocklistPath?: string
+  unifiedJsonBlocklistPath?: string
+  globalCursorIgnorePatterns?: string
+}
+export interface LocalCodebaseFileInfo {
+  unencryptedRelativePath: string
+  hash: string
+  children?: Array<LocalCodebaseFileInfo>
+}
+export declare class DiffClient {
+  constructor()
+  diff(text1: string, text2: string): Array<DiffChunk>
+  /**
+   * use https://docs.rs/diffmatchpatch/latest/diffmatchpatch/struct.DiffMatchPatch.html#method.diff_lines_to_chars
+   * then diff the chars.
+   * then convert back to lines.
+   *
+   * takes in two strings. splits based on newlines.
+   * returns diffs based on lines.
+   */
+  diffLines(text1: string, text2: string): Array<DiffChunk>
+}
+export declare class GitClient {
+  constructor(absoluteRootDirectory: string)
+  getTotalCommitCount(): Promise<number>
+  getCommitVerifyData(commit: string): Promise<VerifyData>
+  throwIfCommitDoesntExist(rootSha: string): Promise<void>
+  getVerifyCommit(): Promise<string>
+  getRepoHeadSha(): Promise<string | null>
+  getCommitChain(hash: string, depth: number, getFiles: CommitChainGetFiles): Promise<Array<CommitData>>
+}
+export declare class GitGraph {
+  constructor(repo: string, globalCursorIgnoreLegacy?: string | undefined | null, globalCursorJsonUnifiedBlocklist?: string | undefined | null)
+  /**
+   * Get a list of all paths in this repository, up to a maximum depth of `max_depth`. The
+   * paths are returned breadth-first, meaning that all paths at depth 0 are returned before any
+   * paths at depth 1, and so on.
+   * The iteration takes into account `gitignore` and `cursorignore` rules.
+   *
+   * @param maxDepth - maximum depth to search for paths. A depth of 0 means only the root path
+   * is returned.
+   * @param limit - maximum number of paths to return. If `None`, all paths are returned.
+   */
+  getBfsPaths(maxDepth: number, limit?: number | undefined | null): Promise<Array<string>>
+  /**
+   * Get a sorted list of relevant paths in this repository, from most relevant to least
+   * relevant. The list is biased towards the current user, using a heuristic that takes into account
+   * the most recently opened/edited files in the editor, which must be passed in as
+   * `lastEditedFiles`. If no recent files are provided, the returned paths will be sorted
+   * based on the global popularity of the files in the repository (i.e. the number of times each
+   * file has been modified throughout the Git history).
+   *
+   * @param lastEditedFiles - list of files that were recently opened/edited in the editor.
+   * @param maxCommits - maximum number of commits to consider when calculating the relevant paths.
+   * @param maxPaths - number of relevant paths to return
+   */
+  getRelevantPaths(lastEditedFiles: Array<string>, maxCommits: number, maxPaths: number): Promise<Array<string>>
+}
+export declare class RepositoryIndexer {
+  constructor(repo: string, blockListLegacy?: string | undefined | null, blockListJson?: string | undefined | null)
+  extractPullRequests(options: JsPullRequestExtractionOptions, callback: (err: Error | null, indexed: IndexedPullRequest[], ignored: number[], progress: IndexedPullRequestProgress) => Promise<boolean>): Promise<ExtractResults>
+  hydratePullRequests(pullRequests: Array<string>, options: JsPullRequestHydrationOptions, callback: (err: Error | null, arg: IndexedPullRequest) => any): Promise<void>
+  getHandshakeData(): Promise<JsHandshakeData | null>
+  verifyCommitOwnership(secret: string, candidates: Array<string>): Promise<JsHandshakeData | null>
+}
+export declare class MerkleClient {
+  constructor(rootPathsMap: Record<string, string>)
+  build(allowIncremental: boolean, config: WalkDirConfig): Promise<void>
   getSubtreeHash(relativePath: string): Promise<string>
   getNumEmbeddableFiles(): Promise<number>
+  getSimhash(): Promise<Float32Array>
+  getImportantPaths(k: number): Promise<Array<string>>
   getAllFiles(): Promise<Array<string>>
   getAllDirFilesToEmbed(absoluteFilePath: string): Promise<Array<string>>
-  getNextFileToEmbed(): Promise<Array<string>>
-  getSpline(absoluteFilePath: string): Promise<Array<string>>
-  getHashesForFiles(files: Array<string>): Promise<Array<string>>
-  updateRootDirectory(rootDirectory: string): void
+  onDidCreate(absoluteFilePath: string): void
+  onDidChange(absoluteFilePath: string): void
+  onDidDelete(absoluteFilePath: string): void
+  getTreeStructure(): Promise<LocalCodebaseFileInfo | null>
+  dispose(): void
 }

package/index.js

@@ -224,14 +224,72 @@ switch (platform) {
         }
         break
       case 'arm':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-arm-musleabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-arm-musleabihf.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-arm-musleabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-arm-gnueabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-arm-gnueabihf.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-arm-gnueabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'riscv64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-riscv64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-riscv64-musl.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-riscv64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'file_service.linux-riscv64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./file_service.linux-riscv64-gnu.node')
+            } else {
+              nativeBinding = require('@anysphere/file-service-linux-riscv64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 's390x':
         localFileExisted = existsSync(
-          join(__dirname, 'file_service.linux-arm-gnueabihf.node')
+          join(__dirname, 'file_service.linux-s390x-gnu.node')
         )
         try {
           if (localFileExisted) {
-            nativeBinding = require('./file_service.linux-arm-gnueabihf.node')
+            nativeBinding = require('./file_service.linux-s390x-gnu.node')
           } else {
-            nativeBinding = require('@anysphere/file-service-linux-arm-gnueabihf')
+            nativeBinding = require('@anysphere/file-service-linux-s390x-gnu')
           }
         } catch (e) {
           loadError = e
@@ -252,6 +310,14 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { MerkleClient } = nativeBinding
+const { DiffType, DiffClient, GitClient, CommitFileStatus, CommitChainGetFiles, GitGraph, RepositoryIndexer, MULTI_ROOT_ABSOLUTE_PATH, MerkleClient } = nativeBinding
 
+module.exports.DiffType = DiffType
+module.exports.DiffClient = DiffClient
+module.exports.GitClient = GitClient
+module.exports.CommitFileStatus = CommitFileStatus
+module.exports.CommitChainGetFiles = CommitChainGetFiles
+module.exports.GitGraph = GitGraph
+module.exports.RepositoryIndexer = RepositoryIndexer
+module.exports.MULTI_ROOT_ABSOLUTE_PATH = MULTI_ROOT_ABSOLUTE_PATH
 module.exports.MerkleClient = MerkleClient

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-eb8b99bf",
+  "version": "0.0.0-eb962576",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -9,15 +9,21 @@
       "additional": [
         "aarch64-apple-darwin",
         "aarch64-pc-windows-msvc",
+        "aarch64-unknown-linux-musl",
+        "x86_64-unknown-linux-musl",
         "universal-apple-darwin",
         "aarch64-unknown-linux-gnu"
       ]
     }
   },
-  "license": "MIT",
+  "files": [
+    "index.js",
+    "index.d.ts"
+  ],
+  "license": "UNLICENSED",
   "devDependencies": {
-    "@napi-rs/cli": "^2.16.2",
-    "ava": "^5.1.1"
+    "@napi-rs/cli": "^2.18.4",
+    "ava": "^6.3.0"
   },
   "ava": {
     "timeout": "3m"
@@ -36,12 +42,14 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-eb8b99bf",
-    "@anysphere/file-service-darwin-x64": "0.0.0-eb8b99bf",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-eb8b99bf",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-eb8b99bf",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-eb8b99bf",
-    "@anysphere/file-service-darwin-universal": "0.0.0-eb8b99bf",
-    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-eb8b99bf"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-eb962576",
+    "@anysphere/file-service-darwin-x64": "0.0.0-eb962576",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-eb962576",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-eb962576",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-eb962576",
+    "@anysphere/file-service-linux-arm64-musl": "0.0.0-eb962576",
+    "@anysphere/file-service-linux-x64-musl": "0.0.0-eb962576",
+    "@anysphere/file-service-darwin-universal": "0.0.0-eb962576",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-eb962576"
   }
 }

package/.yarnrc.yml

@@ -1 +0,0 @@
-nodeLinker: node-modules

package/Cargo.toml

@@ -1,31 +0,0 @@
-[package]
-edition = "2021"
-name = "file_service"
-version = "0.0.0"
-
-[lib]
-crate-type = ["cdylib"]
-
-[dependencies]
-# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2.12.2", default-features = false, features = ["napi4", "async", "tokio_rt"] }
-napi-derive = "2.12.2"
-tokio = { version = "1.32.0", features = ["process", "full"] }
-sha2 = "0.10.7"
-rand = "0.8.5"
-tempfile = "3.8.0"
-anyhow = "1.0.75"
-tonic = "0.9.2"
-prost = "0.11.9"
-tracing = "0.1.37"
-tracing-subscriber = "0.3.17"
-tracing-appender = "0.2.2"
-
-[build-dependencies]
-napi-build = "2.0.1"
-tonic-build = "0.9.2"
-anyhow = "1.0.75"
-glob = "0.3.0"
-
-[profile.release]
-lto = true

package/build.rs

@@ -1,44 +0,0 @@
-use std::path::Path;
-
-extern crate napi_build;
-
-fn main() -> Result<(), anyhow::Error> {
-  napi_build::setup();
-
-  // print the relative path.
-  // let workspace_root = "../../../../../";
-  // let path = std::path::Path::new(workspace_root).canonicalize()?;
-  // let include_path = Path::join(&path, "schema");
-
-  // // let relevant_protos = &[
-	// // 	"aiserver/v1/repository.proto",
-	// // 	"aiserver/v1/symbolic_context.proto",
-	// // 	"aiserver/v1/utils.proto"
-	// // ];
-  // // let proto_paths = relevant_protos
-  // //   .iter()
-  // //   .map(|proto| Path::join(&include_path, proto))
-	// // 	.collect::<Vec<_>>();
-	// let proto_glob = Path::join(&include_path, "aiserver/v1/*.proto");
-  // let relevant_protos: Vec<_> = glob::glob(proto_glob.to_str().expect("Failed to convert path to str"))?
-  //   .filter_map(Result::ok)
-  //   .collect();
-
-  // let proto_paths = relevant_protos
-  //   .iter()
-  //   .map(|proto_path| proto_path.to_str().expect("Failed to convert path to str"))
-  //   .collect::<Vec<_>>();
-  // let includes = &[include_path.to_str().unwrap()];
-
-  // // print the path
-  // println!("cargo:rustc-env=INCLUDE_PATH={}", include_path.display());
-
-  // tonic_build::configure()
-  //   .build_server(false)
-	// 	.build_transport(true)
-	// 	.out_dir("src/proto")
-  //   .compile(&proto_paths, includes)?;
-
-
-	Ok(())
-}

package/src/file_utils.rs

@@ -1,213 +0,0 @@
-// what methods do i want to support here.
-// 1. isInBadDir
-// 2. isBadFile
-// 3. vscode.workspace.asRelativePath
-// 4. vscode.fs.stat
-
-use anyhow::Error;
-use std::path::Path;
-use tokio::fs;
-
-pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
-  let item_path = file_path
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-  let is_bad_dir =
-    item_path.contains("node_modules") || item_path.contains(".git");
-  Ok(is_bad_dir)
-}
-
-pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
-  let item_path = file_path
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-
-  let path = Path::new(item_path);
-  let file_name = path
-    .file_name()
-    .ok_or(anyhow::anyhow!("Failed to get file name"))?
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert file name to string"))?;
-
-  let extension = path
-    .extension()
-    .ok_or(anyhow::anyhow!("Failed to get extension"))?
-    .to_str()
-    .ok_or(anyhow::anyhow!("Failed to convert extension to string"))?;
-
-  match file_name {
-    "package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "composer.lock"
-    | "Gemfile.lock" | "bun.lockb" => {
-      return Err(anyhow::anyhow!("File is just a lock file"));
-    }
-    _ => {}
-  }
-
-  match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" | "lockb" => {
-      return Err(anyhow::anyhow!("File is just a lock file"));
-    }
-    _ => {}
-  }
-
-  if item_path.contains(".git")
-    || item_path.contains(".svn")
-    || item_path.contains(".hg")
-  {
-    return Err(anyhow::anyhow!("File is just a lock file"));
-  }
-
-  let bad_extensions = vec![".exe", ".dll", ".so", ".o", ".bin"];
-  match Path::new(item_path).extension() {
-    Some(extension) => match extension.to_str() {
-      Some(ext_str) => {
-        if bad_extensions.contains(&ext_str) {
-          return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
-        }
-      }
-      None => {
-        return Err(anyhow::anyhow!("Failed to convert extension to string"))
-      }
-    },
-    None => return Err(anyhow::anyhow!("Failed to get extension")),
-  }
-
-  // #[cfg(not(test))]
-  // {
-  let path = Path::new(item_path);
-  for part in path.iter() {
-    match part.to_str() {
-      Some(s) if s.starts_with(".") => {
-        return Err(anyhow::anyhow!("File is hidden"))
-      }
-      _ => {}
-    }
-  }
-  // }
-
-  Ok(())
-}
-
-// implement the buffer above:
-pub async fn is_good_file_runtime_check(
-  file_path: &Path,
-  buffer: &[u8],
-) -> Result<(), Error> {
-  match get_file_size(file_path).await {
-    Ok(size) if size > 2 * 1024 * 1024 => {
-      return Err(anyhow::anyhow!("Buffer is too large"));
-    }
-    Err(e) => return Err(e),
-    _ => {}
-  }
-
-  for &byte in buffer.iter().take(2048) {
-    if byte.is_ascii() {
-      continue;
-    } else {
-      return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
-    }
-  }
-  Ok(())
-}
-
-pub fn as_relative_path(
-  base_path: &Path,
-  file_path: &Path,
-) -> Result<String, Error> {
-  let relative_path = file_path.strip_prefix(base_path)?;
-  Ok(
-    relative_path
-      .to_str()
-      .ok_or(anyhow::anyhow!("Failed to convert relative path to string"))?
-      .to_string(),
-  )
-}
-
-pub async fn get_file_size(file_path: &Path) -> Result<u64, Error> {
-  let metadata = fs::metadata(file_path).await?;
-
-  Ok(metadata.len())
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use std::path::Path;
-  use tokio::io::AsyncWriteExt;
-
-  #[test]
-  fn test_is_in_bad_dir() {
-    let path = Path::new("src/node_modules/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), true);
-
-    let path = Path::new("src/.git/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), true);
-
-    let path = Path::new("src/test.rs");
-    assert_eq!(is_in_bad_dir(&path).unwrap(), false);
-  }
-
-  #[test]
-  fn test_is_good_file() {
-    let path = Path::new("src/test.rs");
-    assert_eq!(is_good_file(&path).is_ok(), true);
-
-    let path = Path::new("src/test.exe");
-    assert_eq!(is_good_file(&path).is_err(), true);
-
-    let path = Path::new("src/.hidden");
-    assert_eq!(is_good_file(&path).is_err(), true);
-  }
-
-  #[tokio::test]
-  async fn test_is_good_file_runtime_check() {
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(b"Hello, world!").await.unwrap();
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_ok(),
-      true
-    );
-    temp_dir.close().unwrap();
-
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_err(),
-      true
-    );
-    temp_dir.close().unwrap();
-  }
-
-  #[test]
-  fn test_as_relative_path() {
-    let base_path = Path::new("/home/user/src");
-    let file_path = Path::new("/home/user/src/test.rs");
-    assert_eq!(as_relative_path(&base_path, &file_path).unwrap(), "test.rs");
-
-    let file_path = Path::new("/home/user/test.rs");
-    assert!(as_relative_path(&base_path, &file_path).is_err());
-  }
-
-  #[tokio::test]
-  async fn test_get_file_size() {
-    let temp_dir = tempfile::tempdir().unwrap();
-    let temp_file_path = temp_dir.path().join("test_file.txt");
-    let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(b"Hello, world!").await.unwrap();
-
-    let size = get_file_size(&temp_file_path).await.unwrap();
-    assert_eq!(size, 13);
-    temp_dir.close().unwrap();
-  }
-}