@hienlh/ppm 0.23.11 → 0.23.12

package/src/services/file-index/file-index-runner.ts

@@ -0,0 +1,169 @@
+/**
+ * Runs index walks on the file index worker, or on this thread when no worker can run.
+ *
+ * A compiled build only has a worker that is one of its entry points (the `build` script,
+ * `scripts/release.sh` and the release workflow all name it, and release-compile-entries.test.ts
+ * keeps them agreeing), and there a worker that cannot load fails with an error event after it
+ * was constructed, not with a throw.
+ * So every walk it was given is re-run here: a missing worker costs the event loop the walk,
+ * which is where every walk ran before, and never costs a caller its answer.
+ */
+import { gzip } from "node:zlib";
+import { promisify } from "node:util";
+import { isCompiledBinary } from "../autostart-generator.ts";
+import type { FileEntry } from "../../types/project.ts";
+import { serializeIndex, walkIndex, type IndexBuild, type IndexFilter } from "./index-walk.ts";
+import { searchIndex, toSearchable, type SearchableIndex, type SearchKind } from "./index-search.ts";
+import type { IndexReply, IndexRequest } from "./file-index-worker.ts";
+
+const gzipAsync = promisify(gzip);
+
+/**
+ * Entries per slice when walking on this thread. An entry costs a few glob tests and two gitignore
+ * checks — ~34 µs on nxsys-workspace — so a slice is about 10 ms, which is as long as a health
+ * probe or a chat stream waits. Counted rather than timed so that how often it yields does not
+ * depend on the machine.
+ */
+const MAIN_THREAD_ENTRIES_PER_SLICE = 256;
+
+/**
+ * How to name the worker so Bun will load it — the same two spellings, for the same reasons, as
+ * `extensionHostWorkerSpec` in extension.service.ts: anchored to this module from source, and
+ * relative to the build's main entry (`src/index.ts`) when compiled.
+ */
+export function fileIndexWorkerSpec(compiled: boolean = isCompiledBinary()): string {
+  return compiled
+    ? "./services/file-index/file-index-worker.ts"
+    : new URL("./file-index-worker.ts", import.meta.url).href;
+}
+
+type WalkRequest = Extract<IndexRequest, { type: "walk" }>;
+type SearchRequest = Extract<IndexRequest, { type: "search" }>;
+
+interface PendingWalk { request: WalkRequest; resolve: (build: IndexBuild) => void; reject: (e: Error) => void }
+interface PendingSearch { request: SearchRequest; build: IndexBuild; resolve: (files: FileEntry[]) => void; reject: (e: Error) => void }
+
+export class FileIndexRunner {
+  private worker: Worker | null = null;
+  private failed = false;
+  private nextId = 1;
+  private readonly walks = new Map<number, PendingWalk>();
+  private readonly searches = new Map<number, PendingSearch>();
+  /** Which walk of each project the worker holds for searching, by the build's hash. */
+  private readonly workerHolds = new Map<string, string>();
+  /** The same, searched on this thread, for when no worker runs. */
+  private readonly heldHere = new Map<string, { hash: string; index: SearchableIndex }>();
+
+  constructor(private readonly spec: string = fileIndexWorkerSpec()) {}
+
+  /** Whether walks now run on this thread, because the worker could not be started or died. */
+  get onMainThread(): boolean {
+    return this.failed;
+  }
+
+  run(rootPath: string, filter: IndexFilter): Promise<IndexBuild> {
+    const worker = this.ensureWorker();
+    if (!worker) return walkHere(rootPath, filter);
+    return new Promise((resolve, reject) => {
+      const request: WalkRequest = { type: "walk", id: this.nextId++, rootPath, filter };
+      this.walks.set(request.id, { request, resolve, reject });
+      worker.postMessage(request);
+    });
+  }
+
+  /**
+   * The best `limit` entries of `build` for `query`, best first (see `searchIndex`). `build` is
+   * the list the caller serves: the worker is handed a copy only when it holds another one —
+   * a walk it finished that the caller dropped, or none, for a project short enough to send.
+   */
+  search(rootPath: string, build: IndexBuild, query: string, kind: SearchKind, limit: number): Promise<FileEntry[]> {
+    const worker = this.ensureWorker();
+    if (!worker) return Promise.resolve(this.searchHere(rootPath, build, query, kind, limit));
+    if (this.workerHolds.get(rootPath) !== build.hash) {
+      worker.postMessage({ type: "seed", rootPath, json: build.json } satisfies IndexRequest);
+      this.workerHolds.set(rootPath, build.hash);
+    }
+    return new Promise((resolve, reject) => {
+      const request: SearchRequest = { type: "search", id: this.nextId++, rootPath, query, kind, limit };
+      this.searches.set(request.id, { request, build, resolve, reject });
+      worker.postMessage(request);
+    });
+  }
+
+  close(): void {
+    this.worker?.terminate();
+    this.worker = null;
+  }
+
+  private ensureWorker(): Worker | null {
+    if (this.worker || this.failed) return this.worker;
+    let worker: Worker;
+    try {
+      worker = new Worker(this.spec, { type: "module" });
+    } catch (e) {
+      this.fail(e as Error);
+      return null;
+    }
+    worker.addEventListener("message", (event: MessageEvent<IndexReply>) => {
+      const reply = event.data;
+      const walk = this.walks.get(reply.id);
+      const search = this.searches.get(reply.id);
+      this.walks.delete(reply.id);
+      this.searches.delete(reply.id);
+      if (!reply.ok) (walk ?? search)?.reject(new Error(reply.error));
+      else if ("build" in reply && walk) {
+        if (reply.searchable) this.workerHolds.set(walk.request.rootPath, reply.build.hash);
+        walk.resolve(reply.build);
+      } else if ("files" in reply && search) search.resolve(reply.files);
+    });
+    worker.addEventListener("error", (event) => this.fail(new Error(event.message)));
+    worker.addEventListener("close", () => {
+      if (this.worker === worker) this.fail(new Error("the worker exited"));
+    });
+    // Its walks are all someone's request; an idle worker must not keep the process alive.
+    (worker as Worker & { unref(): void }).unref();
+    this.worker = worker;
+    return worker;
+  }
+
+  private fail(error: Error): void {
+    if (this.failed) return;
+    this.failed = true;
+    const worker = this.worker;
+    this.worker = null;
+    worker?.terminate();
+    console.warn(`[file-index] walking on the main thread: the index worker failed (${error.message})`);
+    const walks = [...this.walks.values()];
+    const searches = [...this.searches.values()];
+    this.walks.clear();
+    this.searches.clear();
+    for (const { request, resolve, reject } of walks) {
+      walkHere(request.rootPath, request.filter).then(resolve, reject);
+    }
+    for (const { request, build, resolve } of searches) {
+      resolve(this.searchHere(request.rootPath, build, request.query, request.kind, request.limit));
+    }
+  }
+
+  private searchHere(rootPath: string, build: IndexBuild, query: string, kind: SearchKind, limit: number): FileEntry[] {
+    let held = this.heldHere.get(rootPath);
+    if (held?.hash !== build.hash) {
+      const entries = JSON.parse(new TextDecoder().decode(build.json)).data as FileEntry[];
+      held = { hash: build.hash, index: toSearchable(entries) };
+      this.heldHere.set(rootPath, held);
+    }
+    return searchIndex(held.index, query, kind, limit);
+  }
+}
+
+async function walkHere(rootPath: string, filter: IndexFilter): Promise<IndexBuild> {
+  const started = performance.now();
+  const entries = await walkIndex(rootPath, filter, MAIN_THREAD_ENTRIES_PER_SLICE);
+  const { json, hash } = serializeIndex(entries);
+  // zlib's thread pool, not `Bun.gzipSync`: nxsys-workspace's list is 22 MB (see gzip-json.ts).
+  const gzipped = new Uint8Array(await gzipAsync(json));
+  return { json, gzip: gzipped, hash, count: entries.length, walkMs: performance.now() - started };
+}
+
+/** The runner the index service uses. */
+export const fileIndexRunner = new FileIndexRunner();

package/src/services/file-index/file-index-worker.ts

@@ -0,0 +1,73 @@
+/**
+ * The file index worker: walks a project, serialises and gzips the list, on a thread of its own.
+ *
+ * On the main thread the walk of nxsys-workspace (181k entries) cost ~7 s of the event loop in
+ * 10 ms slices, and serialising and gzipping each download another ~120 ms — time every request,
+ * chat stream and terminal waited behind. Here none of it touches the thread that serves them.
+ * The results cross back as transferred buffers, so the main thread copies nothing either.
+ *
+ * It also keeps the lists too long to send to a browser, and searches them (`index-search.ts`).
+ *
+ * It is a second entry point of `bun build --compile`; see `fileIndexWorkerSpec`.
+ */
+import { REMOTE_FILE_SEARCH_FROM_ENTRIES } from "../../shared/file-index-limits.ts";
+import type { FileEntry } from "../../types/project.ts";
+import { serializeIndex, walkIndex, type IndexBuild, type IndexFilter } from "./index-walk.ts";
+import { searchIndex, toSearchable, type SearchableIndex, type SearchKind } from "./index-search.ts";
+
+export type IndexRequest =
+  | { type: "walk"; id: number; rootPath: string; filter: IndexFilter }
+  | { type: "search"; id: number; rootPath: string; query: string; kind: SearchKind; limit: number }
+  /** Hand over a list to search, for a project this thread does not hold one of. No reply. */
+  | { type: "seed"; rootPath: string; json: Uint8Array };
+
+export type IndexReply =
+  /** `searchable`: whether this thread now holds the walked list, ready to search. */
+  | { id: number; ok: true; build: IndexBuild; searchable: boolean }
+  | { id: number; ok: true; files: FileEntry[] }
+  | { id: number; ok: false; error: string };
+
+/**
+ * Entries between two hand-backs of this thread. Nothing here waits on a timer or a socket, so a
+ * slice only has to be short enough for a search to be answered between two of them: 512 entries
+ * are ~18 ms on nxsys-workspace, at ~0.4 s added to its ~6.3 s walk for the yields.
+ */
+const ENTRIES_PER_SLICE = 512;
+
+/** The lists this thread searches, by project root. */
+const searchable = new Map<string, SearchableIndex>();
+
+declare const self: Worker;
+
+self.onmessage = async (event: MessageEvent<IndexRequest>) => {
+  const request = event.data;
+  if (request.type === "seed") {
+    const entries = JSON.parse(new TextDecoder().decode(request.json)).data as FileEntry[];
+    searchable.set(request.rootPath, toSearchable(entries));
+    return;
+  }
+  if (request.type === "search") {
+    const index = searchable.get(request.rootPath);
+    if (!index) {
+      self.postMessage({ id: request.id, ok: false, error: "no list held for this project" } satisfies IndexReply);
+      return;
+    }
+    const files = searchIndex(index, request.query, request.kind, request.limit);
+    self.postMessage({ id: request.id, ok: true, files } satisfies IndexReply);
+    return;
+  }
+  const { id, rootPath, filter } = request;
+  try {
+    const started = performance.now();
+    const entries = await walkIndex(rootPath, filter, ENTRIES_PER_SLICE);
+    const { json, hash } = serializeIndex(entries);
+    const gzip = Bun.gzipSync(json);
+    const build: IndexBuild = { json, gzip, hash, count: entries.length, walkMs: performance.now() - started };
+    // A list short enough to send is searched in the browser, unless it was searched here before.
+    const keep = entries.length >= REMOTE_FILE_SEARCH_FROM_ENTRIES || searchable.has(rootPath);
+    if (keep) searchable.set(rootPath, toSearchable(entries));
+    self.postMessage({ id, ok: true, build, searchable: keep } satisfies IndexReply, [json.buffer, gzip.buffer]);
+  } catch (e) {
+    self.postMessage({ id, ok: false, error: (e as Error).message } satisfies IndexReply);
+  }
+};

package/src/services/file-index/index-search.ts

@@ -0,0 +1,152 @@
+/**
+ * Searching a project's file list where it is held — on the index worker — for a list too long
+ * to send to the browser (`REMOTE_FILE_SEARCH_FROM_ENTRIES`). It ranks exactly as the palette
+ * ranks a list it holds itself (`score-file-search.ts`), so a project does not start searching
+ * differently for crossing the threshold.
+ *
+ * Two things make a keystroke cheaper here than the palette's own loop over the same list. Only
+ * the best `limit` are ordered (`BestOf`). And a query that extends a recent one only looks at
+ * what that one matched: every tier is a match of the query as a subsequence of the path, which
+ * each prefix of the query matches too.
+ */
+import type { FileEntry } from "../../types/project.ts";
+import { compareScores, getFilename, scoreFileSearchFast, scoreWordsFast, type FileSearchScore } from "../../web/lib/score-file-search.ts";
+
+/** Files only (the palette, the compare picker), or directories too (the chat's @-picker). */
+export type SearchKind = "file" | "all";
+
+/** A list prepared for searching: every path lowercased once, instead of once per keystroke. */
+export interface SearchableIndex {
+  entries: FileEntry[];
+  pathLower: string[];
+  nameLower: string[];
+  depth: Uint16Array;
+  /** The last few queries and what each matched, the most recent last. */
+  recent: { kind: SearchKind; query: string; matches: Int32Array }[];
+}
+
+const RECENT_QUERIES = 8;
+
+export function toSearchable(entries: FileEntry[]): SearchableIndex {
+  const n = entries.length;
+  const pathLower = new Array<string>(n);
+  const nameLower = new Array<string>(n);
+  const depth = new Uint16Array(n);
+  for (let i = 0; i < n; i++) {
+    const path = entries[i]!.path;
+    const lower = path.toLowerCase();
+    pathLower[i] = lower;
+    nameLower[i] = getFilename(lower);
+    let segments = 1;
+    for (let at = path.indexOf("/"); at >= 0; at = path.indexOf("/", at + 1)) segments++;
+    depth[i] = segments;
+  }
+  return { entries, pathLower, nameLower, depth, recent: [] };
+}
+
+/**
+ * The best `limit` entries of `kind` for `query`, best first — the order the palette would put
+ * them in. A blank query answers the first `limit` in list order, which is what a picker shows
+ * before anything is typed.
+ */
+export function searchIndex(index: SearchableIndex, query: string, kind: SearchKind, limit: number): FileEntry[] {
+  const { entries } = index;
+  if (!query.trim()) {
+    if (kind === "all") return entries.slice(0, limit);
+    const first: FileEntry[] = [];
+    for (let i = 0; i < entries.length && first.length < limit; i++) {
+      if (entries[i]!.type === "file") first.push(entries[i]!);
+    }
+    return first;
+  }
+
+  // As the palette reads it: `./` and `../` name the project root, which the paths leave out.
+  const q = query.toLowerCase().replace(/^\.\.?\//, "");
+  const words = q.includes(" ") ? q.split(/\s+/).filter(Boolean) : null;
+  const narrowed = narrowFrom(index, kind, q);
+  const best = new BestOf(limit);
+  const matches = new Int32Array(narrowed ? narrowed.length : entries.length);
+  let matched = 0;
+  const visit = (i: number) => {
+    const score = words
+      ? scoreWordsFast(words, index.nameLower[i]!, index.pathLower[i]!, entries[i]!.name.length, index.depth[i]!)
+      : scoreFileSearchFast(q, index.nameLower[i]!, index.pathLower[i]!, entries[i]!.name.length, index.depth[i]!);
+    if (!score) return;
+    best.offer(i, score);
+    matches[matched++] = i;
+  };
+  if (narrowed) {
+    for (let k = 0; k < narrowed.length; k++) visit(narrowed[k]!);
+  } else {
+    for (let i = 0; i < entries.length; i++) {
+      if (kind === "all" || entries[i]!.type === "file") visit(i);
+    }
+  }
+  remember(index, { kind, query: q, matches: matches.slice(0, matched) });
+  return best.sorted().map((i) => entries[i]!);
+}
+
+interface Ranked { i: number; score: FileSearchScore }
+
+/** The palette's order, which is a stable sort: equal scores stay in list order. */
+function rank(a: Ranked, b: Ranked): number {
+  return compareScores(a.score, b.score) || a.i - b.i;
+}
+
+/**
+ * The best `limit` of what it is offered, kept in a heap with the worst on top — so each match
+ * costs at most a comparison with it. Sorting every match instead was 25 ms of the 46 that a
+ * one-letter query took on nxsys-workspace, most of it spent ordering what nobody would see.
+ */
+class BestOf {
+  private readonly heap: Ranked[] = [];
+  constructor(private readonly limit: number) {}
+
+  offer(i: number, score: FileSearchScore): void {
+    const { heap } = this;
+    if (heap.length < this.limit) {
+      heap.push({ i, score });
+      let at = heap.length - 1;
+      while (at > 0) {
+        const parent = (at - 1) >> 1;
+        if (rank(heap[at]!, heap[parent]!) <= 0) break;
+        [heap[at], heap[parent]] = [heap[parent]!, heap[at]!];
+        at = parent;
+      }
+      return;
+    }
+    if (this.limit === 0 || rank({ i, score }, heap[0]!) >= 0) return;
+    heap[0] = { i, score };
+    let at = 0;
+    for (;;) {
+      const left = at * 2 + 1;
+      const right = left + 1;
+      let worst = at;
+      if (left < heap.length && rank(heap[left]!, heap[worst]!) > 0) worst = left;
+      if (right < heap.length && rank(heap[right]!, heap[worst]!) > 0) worst = right;
+      if (worst === at) break;
+      [heap[at], heap[worst]] = [heap[worst]!, heap[at]!];
+      at = worst;
+    }
+  }
+
+  sorted(): number[] {
+    return [...this.heap].sort(rank).map((r) => r.i);
+  }
+}
+
+/** What the longest recent query that `q` extends matched: all `q` can match is among it. */
+function narrowFrom(index: SearchableIndex, kind: SearchKind, q: string): Int32Array | null {
+  let from: SearchableIndex["recent"][number] | null = null;
+  for (const recent of index.recent) {
+    if (recent.kind !== kind || !recent.query || !q.startsWith(recent.query)) continue;
+    if (!from || recent.query.length > from.query.length) from = recent;
+  }
+  return from?.matches ?? null;
+}
+
+function remember(index: SearchableIndex, search: SearchableIndex["recent"][number]): void {
+  const recent = index.recent.filter((r) => r.kind !== search.kind || r.query !== search.query);
+  recent.push(search);
+  index.recent = recent.slice(-RECENT_QUERIES);
+}

package/src/services/file-index/index-walk.ts

@@ -0,0 +1,109 @@
+/**
+ * The walk behind a project's flat file index, and what is served from it.
+ *
+ * Runs on the file index worker (`file-index-worker.ts`), or on the main thread where no worker
+ * could be started. It must stay free of any import that reaches the config store or the
+ * database: the filter arrives already resolved.
+ */
+import { existsSync, readdirSync, readFileSync, type Dirent } from "node:fs";
+import { join, relative } from "node:path";
+import ignore, { type Ignore } from "ignore";
+import type { FileEntry } from "../../types/project.ts";
+import { ok } from "../../types/api.ts";
+import { matchesGlob } from "../file-glob.ts";
+
+/** A project's filter, resolved from the config store by whoever asks for the walk. */
+export interface IndexFilter {
+  /** filesExclude + searchExclude: hard excludes, never listed. */
+  exclude: string[];
+  /** Whether gitignored files are flagged `isIgnored` (they are still listed). */
+  useIgnoreFiles: boolean;
+}
+
+/**
+ * A walk, ready to serve: the whole `/files/index` response body — envelope included, so the
+ * route sends it as it is — plain and gzipped, and a hash that says whether the list moved.
+ */
+export interface IndexBuild {
+  json: Uint8Array<ArrayBuffer>;
+  gzip: Uint8Array<ArrayBuffer>;
+  /** Of `json`. Equal hashes mean the same paths, types and ignore flags in the same order. */
+  hash: string;
+  count: number;
+  walkMs: number;
+}
+
+export function loadGitignore(projectPath: string): Ignore {
+  const ig = ignore();
+  const gitignorePath = join(projectPath, ".gitignore");
+  if (existsSync(gitignorePath)) {
+    try {
+      const content = readFileSync(gitignorePath, "utf-8");
+      ig.add(content);
+    } catch { /* unreadable — skip */ }
+  }
+  return ig;
+}
+
+/**
+ * Flat list of everything under `rootPath` that the filter keeps, depth-first. Hands its thread
+ * back every `entriesPerSlice` entries, so whatever else runs there is not kept waiting.
+ */
+export async function walkIndex(rootPath: string, filter: IndexFilter, entriesPerSlice: number): Promise<FileEntry[]> {
+  const ig = filter.useIgnoreFiles ? loadGitignore(rootPath) : null;
+  const allExclude = filter.exclude;
+  const results: FileEntry[] = [];
+
+  // Depth-first on an explicit stack, in the order the recursive walk it replaces produced.
+  const stack: { dirPath: string; dirEntries: Dirent[]; next: number }[] = [];
+  const enter = (dirPath: string) => {
+    try { stack.push({ dirPath, dirEntries: readdirSync(dirPath, { withFileTypes: true }), next: 0 }); }
+    catch { /* unreadable — skip */ }
+  };
+  enter(rootPath);
+
+  let sinceYield = 0;
+  while (stack.length > 0) {
+    const frame = stack[stack.length - 1]!;
+    const entry = frame.dirEntries[frame.next++];
+    if (!entry) {
+      stack.pop();
+      continue;
+    }
+    if (++sinceYield >= entriesPerSlice) {
+      sinceYield = 0;
+      await new Promise<void>((r) => setTimeout(r, 0));
+    }
+
+    const fullPath = join(frame.dirPath, entry.name);
+    const relPath = relative(rootPath, fullPath);
+    const relPosix = relPath.split("\\").join("/");
+
+    // Apply glob exclusion (check full relative path and bare entry name)
+    // These are HARD excludes — .git, node_modules, dist, etc.
+    if (matchesGlob(relPosix, allExclude)) continue;
+    if (matchesGlob(entry.name, allExclude)) continue;
+
+    // Apply gitignore rules — SOFT exclude only (mark with isIgnored flag).
+    // Huge dirs like node_modules/dist/build are already hard-excluded by glob above.
+    let isIgnored = false;
+    if (ig) {
+      const checkPath = entry.isDirectory() ? `${relPosix}/` : relPosix;
+      isIgnored = ig.ignores(checkPath) || ig.ignores(relPosix);
+    }
+
+    if (entry.isDirectory()) {
+      results.push({ path: relPosix, name: entry.name, type: "directory" });
+      enter(fullPath);
+    } else {
+      results.push({ path: relPosix, name: entry.name, type: "file", ...(isIgnored && { isIgnored: true }) });
+    }
+  }
+  return results;
+}
+
+/** The response body for `entries` and a hash of it; gzipping is left to the caller's thread. */
+export function serializeIndex(entries: FileEntry[]): { json: Uint8Array<ArrayBuffer>; hash: string } {
+  const json = new TextEncoder().encode(JSON.stringify(ok(entries)));
+  return { json, hash: Bun.hash(json).toString(16) };
+}