@takara-ai/miru-code 1.9.0 → 1.10.0

package/src/mcp/stdio.ts

@@ -45,6 +45,21 @@ export class StdioTransport {
   private buffer = "";
   private started = false;
   private abortController: AbortController | null = null;
+  private readonly inputStream: () => ReadableStream<Uint8Array<ArrayBufferLike>>;
+  private readonly writeText: (text: string) => Promise<unknown>;
+
+  constructor(
+    options: {
+      inputStream?: () => ReadableStream<Uint8Array<ArrayBufferLike>>;
+      writeText?: (text: string) => Promise<unknown>;
+      stdin?: { stream: () => ReadableStream<Uint8Array<ArrayBufferLike>> };
+      bunWrite?: (destination: typeof Bun.stdout, text: string) => Promise<unknown>;
+    } = {},
+  ) {
+    this.inputStream = options.inputStream ?? (() => (options.stdin ?? Bun.stdin).stream());
+    this.writeText =
+      options.writeText ?? ((text) => (options.bunWrite ?? Bun.write)(Bun.stdout, text));
+  }
 
   onmessage?: (message: JsonRpcMessage) => void | Promise<void>;
   onerror?: (error: Error) => void;
@@ -56,7 +71,7 @@ export class StdioTransport {
     }
     this.started = true;
     this.abortController = new AbortController();
-    const reader = Bun.stdin.stream().getReader();
+    const reader = this.inputStream().getReader();
     const decoder = new TextDecoder();
 
     try {
@@ -85,7 +100,7 @@ export class StdioTransport {
 
   async send(message: JsonRpcMessage): Promise<void> {
     const line = `${JSON.stringify(message)}\n`;
-    await Bun.write(Bun.stdout, line);
+    await this.writeText(line);
   }
 
   private async append(chunk: string): Promise<void> {

package/src/miru-index.ts

@@ -26,6 +26,31 @@ import { chunkKey, chunkToDict, defaultContentTypes } from "./types.ts";
 import { computeSourceCacheKey, isGitUrl } from "./utils.ts";
 import { indexCacheEpoch } from "./version.ts";
 
+interface FactoryDependencies {
+  getValidatedCache: typeof getValidatedCache;
+  resolveEmbeddingModel: typeof resolveEmbeddingModel;
+  getEmbeddingBackend: (model: string) => EmbeddingBackend;
+  createIndexFromPath: typeof createIndexFromPath;
+  cloneGitRepository: typeof cloneGitRepository;
+  findIndexCachePath: typeof findIndexCachePath;
+  loadCachedIndex: typeof loadCachedIndex;
+  saveBuiltIndex: (index: MiruIndex, path: string) => Promise<void>;
+}
+
+function factoryDependencies(overrides: Partial<FactoryDependencies> = {}): FactoryDependencies {
+  return {
+    getValidatedCache,
+    resolveEmbeddingModel,
+    getEmbeddingBackend,
+    createIndexFromPath,
+    cloneGitRepository,
+    findIndexCachePath,
+    loadCachedIndex,
+    saveBuiltIndex: (index, path) => index.save(path),
+    ...overrides,
+  };
+}
+
 /**
  * In-memory search index over a codebase: BM25 keyword scores plus semantic
  * embeddings, with optional disk cache for reuse across runs.
@@ -171,18 +196,20 @@ export class MiruIndex {
     content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
     ref?: string | null,
+    dependencies?: Partial<FactoryDependencies>,
   ): Promise<MiruIndex> {
+    const deps = factoryDependencies(dependencies);
     const cacheKey = computeSourceCacheKey(url, ref);
-    const model = embeddingModel ?? resolveEmbeddingModel();
-    const cached = await getValidatedCache(cacheKey, model, content);
+    const model = embeddingModel ?? deps.resolveEmbeddingModel();
+    const cached = await deps.getValidatedCache(cacheKey, model, content);
     if (cached) {
-      return MiruIndex.loadFromDisk(cached, model);
+      return MiruIndex.loadFromDisk(cached, model, deps);
     }
 
-    const cloneDir = await cloneGitRepository(url, ref);
+    const cloneDir = await deps.cloneGitRepository(url, ref);
     try {
-      const embeddings = getEmbeddingBackend(model);
-      const { bm25, semantic, chunks } = await createIndexFromPath(
+      const embeddings = deps.getEmbeddingBackend(model);
+      const { bm25, semantic, chunks } = await deps.createIndexFromPath(
         cloneDir,
         embeddings,
         content,
@@ -198,7 +225,7 @@ export class MiruIndex {
         root: null,
         content,
       });
-      await index.save(findIndexCachePath(cacheKey));
+      await deps.saveBuiltIndex(index, deps.findIndexCachePath(cacheKey));
       return index;
     } finally {
       await rm(cloneDir, { recursive: true, force: true });
@@ -211,11 +238,12 @@ export class MiruIndex {
     content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
     ref?: string | null,
+    dependencies?: Partial<FactoryDependencies>,
   ): Promise<MiruIndex> {
     if (isGitUrl(source)) {
-      return MiruIndex.fromGit(source, content, embeddingModel, ref);
+      return MiruIndex.fromGit(source, content, embeddingModel, ref, dependencies);
     }
-    return MiruIndex.fromPath(source, content, embeddingModel);
+    return MiruIndex.fromPath(source, content, embeddingModel, dependencies);
   }
 
   /**
@@ -229,7 +257,9 @@ export class MiruIndex {
     path: string,
     content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
+    dependencies?: Partial<FactoryDependencies>,
   ): Promise<MiruIndex> {
+    const deps = factoryDependencies(dependencies);
     const resolved = resolve(path);
     try {
       const st = await Bun.file(resolved).stat();
@@ -243,14 +273,14 @@ export class MiruIndex {
       throw new Error(`Path does not exist: ${path}`);
     }
 
-    const model = embeddingModel ?? resolveEmbeddingModel();
-    const cached = await getValidatedCache(resolved, model, content);
+    const model = embeddingModel ?? deps.resolveEmbeddingModel();
+    const cached = await deps.getValidatedCache(resolved, model, content);
     if (cached) {
-      return MiruIndex.loadFromDisk(cached, model);
+      return MiruIndex.loadFromDisk(cached, model, deps);
     }
 
-    const embeddings = getEmbeddingBackend(model);
-    const { bm25, semantic, chunks } = await createIndexFromPath(
+    const embeddings = deps.getEmbeddingBackend(model);
+    const { bm25, semantic, chunks } = await deps.createIndexFromPath(
       resolved,
       embeddings,
       content,
@@ -274,15 +304,20 @@ export class MiruIndex {
    * Validates embedding model compatibility via cache metadata. Sets
    * `loadedFromDisk` so callers can avoid redundant cache writes.
    */
-  static async loadFromDisk(path: string, embeddingModel?: string): Promise<MiruIndex> {
-    const model = embeddingModel ?? resolveEmbeddingModel();
-    const { bm25, semantic, chunks, metadata } = await loadCachedIndex(path);
+  static async loadFromDisk(
+    path: string,
+    embeddingModel?: string,
+    dependencies?: Partial<FactoryDependencies>,
+  ): Promise<MiruIndex> {
+    const deps = factoryDependencies(dependencies);
+    const model = embeddingModel ?? deps.resolveEmbeddingModel();
+    const { bm25, semantic, chunks, metadata } = await deps.loadCachedIndex(path);
     const content = (metadata.content_type as ContentType[]) ?? defaultContentTypes();
     const root = metadata.root_path ? String(metadata.root_path) : null;
     const storedFileMtimes = (metadata.file_mtimes as Record<string, number>) ?? {};
 
     return new MiruIndex({
-      embeddings: getEmbeddingBackend(model),
+      embeddings: deps.getEmbeddingBackend(model),
       bm25Index: bm25,
       semanticIndex: semantic,
       chunks,
@@ -387,6 +422,7 @@ export class MiruIndex {
       embeddings: this.embeddings,
       chunks: this.chunksInternal,
       semanticIndex: this.semanticIndex,
+      bm25: this.bm25Index,
       relativePaths,
     });
 

package/src/prompt.ts

@@ -62,22 +62,49 @@ export function applyHiddenPromptChar(
 }
 
 /** Visible-input prompt (not for secrets — use promptHidden for those). */
-export async function promptText(message: string, defaultValue = ""): Promise<string> {
-  const rl = readline.createInterface({ input, output });
+export async function promptText(
+  message: string,
+  defaultValue = "",
+  createInterface: typeof createPromptInterface = createPromptInterface,
+): Promise<string> {
+  const rl = createInterface({ input, output });
+  return promptTextWith(
+    message,
+    defaultValue,
+    (question) => rl.question(question),
+    () => rl.close(),
+  );
+}
+
+export function createPromptInterface(
+  options: { input: typeof input; output: typeof output },
+  factory: typeof readline.createInterface = readline.createInterface,
+): Pick<readline.Interface, "question" | "close"> {
+  return factory(options);
+}
+
+export async function promptTextWith(
+  message: string,
+  defaultValue: string,
+  ask: (question: string) => Promise<string>,
+  close: () => void = () => {},
+): Promise<string> {
   try {
     const suffix = defaultValue ? ` (${defaultValue})` : "";
-    const answer = (await rl.question(`${message}${suffix}: `)).trim();
+    const answer = (await ask(`${message}${suffix}: `)).trim();
     return answer || defaultValue;
   } finally {
-    rl.close();
+    close();
   }
 }
 
 export async function promptHidden(
   message: string,
-  stream: NodeJS.WriteStream = output,
+  stream: { write(text: string): unknown } = output,
+  io: { input?: HiddenPromptInput; exit?: (code: number) => void } = {},
 ): Promise<string> {
-  if (!input.isTTY) {
+  const inputStream = io.input ?? input;
+  if (!inputStream.isTTY) {
     throw new Error(
       "Cannot prompt for API key: stdin is not a TTY. Run `miru setup --key YOUR_KEY` or set TAKARA_API_KEY.",
     );
@@ -85,9 +112,9 @@ export async function promptHidden(
 
   return new Promise((resolve, reject) => {
     stream.write(message);
-    input.setRawMode?.(true);
-    input.resume();
-    input.setEncoding("utf8");
+    inputStream.setRawMode?.(true);
+    inputStream.resume();
+    inputStream.setEncoding("utf8");
 
     let state = createHiddenPromptState();
 
@@ -100,7 +127,7 @@ export async function promptHidden(
           cleanup();
           stream.write("\n");
           reject(new Error("Setup cancelled."));
-          process.exit(130);
+          (io.exit ?? process.exit)(130);
         }
 
         if (result.submit) {
@@ -117,11 +144,21 @@ export async function promptHidden(
     };
 
     const cleanup = () => {
-      input.setRawMode?.(false);
-      input.pause();
-      input.removeListener("data", onData);
+      inputStream.setRawMode?.(false);
+      inputStream.pause();
+      inputStream.removeListener("data", onData);
     };
 
-    input.on("data", onData);
+    inputStream.on("data", onData);
   });
 }
+
+interface HiddenPromptInput {
+  isTTY?: boolean;
+  setRawMode?: (mode: boolean) => void;
+  resume(): void;
+  pause(): void;
+  setEncoding(encoding: "utf8"): unknown;
+  on(event: "data", listener: (chunk: string) => void): unknown;
+  removeListener(event: "data", listener: (chunk: string) => void): unknown;
+}

package/src/ranking/boosting.ts

@@ -209,12 +209,11 @@ function boostSymbolDefinitions(
     const stem = basename(chunk.file_path)
       .replace(/\.[^.]+$/, "")
       .toLowerCase();
-    if (![...names].some((n) => stemMatches(stem, n.toLowerCase()))) {
-      continue;
-    }
-    const tier = definitionTier(chunk, names, boostUnit);
-    if (tier) {
-      boosted.set(key, tier);
+    if ([...names].some((n) => stemMatches(stem, n.toLowerCase()))) {
+      const tier = definitionTier(chunk, names, boostUnit);
+      if (tier) {
+        boosted.set(key, tier);
+      }
     }
   }
 }

package/src/ranking/location.ts

@@ -222,16 +222,12 @@ export function boostExactStemMatches(
       if (GENERIC_STEM_DENY.has(keyword)) {
         continue;
       }
-      if (preferImplementation && stem === keyword && !IMPLEMENTATION_HINTS.has(stem)) {
-        continue;
-      }
-      const keywordNorm = keyword.replace(/[-_]/g, "");
-      if (stem === keyword || stemNorm === keywordNorm) {
-        boosted.set(key, (boosted.get(key) ?? 0) + boost);
-        break;
-      }
-      const parts = keyword.split("-").filter((p) => p.length > 2);
-      if (parts.length > 1 && parts.some((p) => p === stem)) {
+      if (!(preferImplementation && stem === keyword && !IMPLEMENTATION_HINTS.has(stem))) {
+        const keywordNorm = keyword.replace(/[-_]/g, "");
+        if (stem === keyword || stemNorm === keywordNorm) {
+          boosted.set(key, (boosted.get(key) ?? 0) + boost);
+          break;
+        }
       }
     }
   }

package/src/ranking/penalties.ts

@@ -72,7 +72,7 @@ export function rerankTopk(
   const ranked = [...penalised.entries()].sort((a, b) => b[1] - a[1]);
 
   const fileSelected = new Map<string, number>();
-  const selected: { score: number; key: string }[] = [];
+  const selected: { score: number; key: string; chunk: Chunk }[] = [];
   let minSelected = Infinity;
 
   for (const [key, penScore] of ranked) {
@@ -81,31 +81,23 @@ export function rerankTopk(
       continue;
     }
 
-    if (selected.length >= topK && penScore <= minSelected) {
-      break;
-    }
-
-    const already = fileSelected.get(chunk.file_path) ?? 0;
-    let effScore = penScore;
-    if (already >= FILE_SATURATION_THRESHOLD) {
-      const excess = already - FILE_SATURATION_THRESHOLD + 1;
-      effScore *= tuning.fileSaturationDecay ** excess;
-    }
+    if (selected.length < topK || penScore > minSelected) {
+      const already = fileSelected.get(chunk.file_path) ?? 0;
+      let effScore = penScore;
+      if (already >= FILE_SATURATION_THRESHOLD) {
+        const excess = already - FILE_SATURATION_THRESHOLD + 1;
+        effScore *= tuning.fileSaturationDecay ** excess;
+      }
 
-    selected.push({ score: effScore, key });
-    fileSelected.set(chunk.file_path, already + 1);
+      selected.push({ score: effScore, key, chunk });
+      fileSelected.set(chunk.file_path, already + 1);
 
-    if (selected.length >= topK) {
-      minSelected = Math.min(...selected.map((s) => s.score));
+      if (selected.length >= topK) {
+        minSelected = Math.min(...selected.map((s) => s.score));
+      }
     }
   }
 
   selected.sort((a, b) => b.score - a.score);
-  return selected.slice(0, topK).flatMap(({ key, score }) => {
-    const chunk = chunksByKey.get(key);
-    if (!chunk) {
-      return [];
-    }
-    return [[chunk, score] as [Chunk, number]];
-  });
+  return selected.slice(0, topK).map(({ chunk, score }) => [chunk, score]);
 }

package/src/search.ts

@@ -137,11 +137,9 @@ export async function hybridSearch(options: {
 
   const allKeys = new Set([...normalizedSemantic.keys(), ...normalizedBm25.keys()]);
   const sortedKeys = [...allKeys].sort((a, b) => {
-    const ca = chunksByKey.get(a);
-    const cb = chunksByKey.get(b);
-    if (!ca || !cb) {
-      return 0;
-    }
+    // Every score key was produced from a chunk in this query.
+    const ca = chunksByKey.get(a) as Chunk;
+    const cb = chunksByKey.get(b) as Chunk;
     return ca.start_line - cb.start_line;
   });
 
@@ -164,13 +162,7 @@ export async function hybridSearch(options: {
   return [...combinedScores.entries()]
     .sort((a, b) => b[1] - a[1])
     .slice(0, topK)
-    .flatMap(([key, score]) => {
-      const chunk = chunksByKey.get(key);
-      if (!chunk) {
-        return [];
-      }
-      return [{ chunk, score }];
-    });
+    .map(([key, score]) => ({ chunk: chunksByKey.get(key) as Chunk, score }));
 }
 
 export async function searchSemanticOnly(options: {

package/src/setup.ts

@@ -123,25 +123,27 @@ export function parseSetupCliArgs(rest: string[]): {
   return { args };
 }
 
-async function promptSageMakerArn(): Promise<string> {
-  while (true) {
-    const arn = await promptText("SageMaker endpoint ARN");
-    if (!arn) {
+async function promptSageMakerArn(promptTextImpl: typeof promptText = promptText): Promise<string> {
+  let arn = "";
+  while (!arn) {
+    const candidate = await promptTextImpl("SageMaker endpoint ARN");
+    if (!candidate) {
       fail("Endpoint ARN cannot be empty.");
       continue;
     }
     try {
-      parseSageMakerEndpointArn(arn);
-      return arn;
+      parseSageMakerEndpointArn(candidate);
+      arn = candidate;
     } catch (err) {
       fail(err instanceof Error ? err.message : String(err));
     }
   }
+  return arn;
 }
 
-async function promptAwsProfile(): Promise<string> {
+async function promptAwsProfile(promptTextImpl: typeof promptText = promptText): Promise<string> {
   while (true) {
-    const profile = await promptText(
+    const profile = await promptTextImpl(
       "AWS profile name (must already exist in ~/.aws)",
       process.env.AWS_PROFILE || "miru",
     );
@@ -152,11 +154,20 @@ async function promptAwsProfile(): Promise<string> {
   }
 }
 
+export interface SageMakerSetupDependencies {
+  promptText?: typeof promptText;
+  canPromptForCredentials?: typeof canPromptForCredentials;
+  validateConnection?: typeof validateSageMakerConnection;
+}
+
 /**
  * Miru only ever inherits AWS credentials — it never creates, writes, or rotates an
  * AWS profile. Set one up yourself first (e.g. `aws configure --profile miru`).
  */
-export async function runSageMakerSetup(options: RunSetupOptions = {}): Promise<RunSetupResult> {
+export async function runSageMakerSetup(
+  options: RunSetupOptions = {},
+  dependencies: SageMakerSetupDependencies = {},
+): Promise<RunSetupResult> {
   writeStdout("");
   printBrandBanner(process.stderr);
   divider("─", 48, process.stderr);
@@ -168,18 +179,19 @@ export async function runSageMakerSetup(options: RunSetupOptions = {}): Promise<
   );
   writeStdout("");
 
-  const arnInput = options.sagemakerArn ?? (await promptSageMakerArn());
+  const promptTextImpl = dependencies.promptText ?? promptText;
+  const arnInput = options.sagemakerArn ?? (await promptSageMakerArn(promptTextImpl));
   const parsed = parseSageMakerEndpointArn(arnInput);
 
   let profile = options.profile;
   if (!profile) {
-    if (!canPromptForCredentials()) {
+    if (!(dependencies.canPromptForCredentials ?? canPromptForCredentials)()) {
       throw new Error(
         "An AWS profile name is required. Pass --profile <name>, or run " +
           "`miru setup --sagemaker` interactively.",
       );
     }
-    profile = await promptAwsProfile();
+    profile = await promptAwsProfile(promptTextImpl);
   }
 
   await beginModeSwitch("sagemaker");
@@ -197,7 +209,7 @@ export async function runSageMakerSetup(options: RunSetupOptions = {}): Promise<
       truncate: true,
       truncationDirection: "Right",
     };
-    const result = await validateSageMakerConnection(config);
+    const result = await (dependencies.validateConnection ?? validateSageMakerConnection)(config);
     if (!result.valid) {
       spinner.stop();
       throw new Error(result.message);

package/src/snippet.ts

@@ -79,7 +79,7 @@ const RUBY_OR_ELIXIR_BODY_RE = /^\s*def\w*\b|^\s*defmodule\b/;
 const MAX_DECLARATION_LOOKBACK = 40;
 const INDENTED_BLOCK_LANGUAGES = new Set(["python", "haskell", "ocaml"]);
 const END_BLOCK_LANGUAGES = new Set(["ruby", "elixir"]);
-const TAG_LANGUAGES = new Set(["html", "embeddedtemplate", "xml", "vue", "svelte"]);
+const TAG_LANGUAGES = new Set(["html", "embeddedtemplate", "xml", "vue", "svelte", "astro"]);
 const BRACE_BLOCK_LANGUAGES = new Set([
   "bash",
   "c",
@@ -192,16 +192,14 @@ function completeHtmlTag(
   for (let i = Math.max(0, start - MAX_DECLARATION_LOOKBACK); i <= anchor; i++) {
     const line = lines[i] ?? "";
     for (const match of line.matchAll(HTML_TAG_RE)) {
-      const name = match[1]?.toLowerCase();
-      if (!name) {
-        continue;
-      }
+      const name = (match[1] as string).toLowerCase();
       if (match[0].startsWith("</")) {
-        for (let j = openTags.length - 1; j >= 0; j--) {
-          if (openTags[j]?.name === name) {
-            openTags.splice(j, 1);
-            break;
-          }
+        const matchingIndex = openTags.reduce(
+          (last, tag, index) => (tag.name === name ? index : last),
+          -1,
+        );
+        if (matchingIndex >= 0) {
+          openTags.splice(matchingIndex, 1);
         }
       } else if (!match[0].endsWith("/>") && !HTML_VOID_TAGS.has(name)) {
         openTags.push({ name, start: i });
@@ -374,10 +372,6 @@ function compactSupportingSnippet(snippet: SnippetResult, maxLines: number): Sni
 /** Pick the 0-based line index inside `content` that best matches the query. */
 export function anchorLineOffset(content: string, query: string): number {
   const lines = content.split("\n");
-  if (lines.length === 0) {
-    return 0;
-  }
-
   const terms = queryMatchTerms(query);
   let bestIndex = Math.floor(lines.length / 2);
   let bestScore = -1;
@@ -401,18 +395,6 @@ export function trimChunkToSnippet(
   anchorLine?: number,
 ): SnippetResult {
   const lines = chunk.content.split("\n");
-  if (lines.length === 0) {
-    return {
-      chunk,
-      meta: {
-        truncated: false,
-        anchor_line: chunk.start_line,
-        full_start_line: chunk.start_line,
-        full_end_line: chunk.end_line,
-      },
-    };
-  }
-
   const queryAnchorOffset = anchorLineOffset(chunk.content, query);
   const anchorOffset =
     anchorLine == null

package/src/spinner.ts

@@ -8,11 +8,14 @@ export class Spinner {
   private frame = 0;
   private belowLines = 0;
 
-  constructor(private readonly message: string) {}
+  constructor(
+    private readonly message: string,
+    private readonly stderr: SpinnerOutput = process.stderr,
+  ) {}
 
   start(): void {
-    if (!process.stderr.isTTY) {
-      process.stderr.write(`${this.message}...\n`);
+    if (!this.stderr.isTTY) {
+      this.stderr.write(`${this.message}...\n`);
       return;
     }
     this.draw();
@@ -20,17 +23,17 @@ export class Spinner {
   }
 
   follow(line: string): void {
-    if (!process.stderr.isTTY) {
-      process.stderr.write(`${line}\n`);
+    if (!this.stderr.isTTY) {
+      this.stderr.write(`${line}\n`);
       return;
     }
-    const cols = process.stderr.columns || 80;
+    const cols = this.stderr.columns || 80;
     const rows = line.split("\n").reduce((count, part) => {
       return count + Math.max(1, Math.ceil(displayWidth(part) / cols));
     }, 0);
-    process.stderr.write(`\n${line}`);
+    this.stderr.write(`\n${line}`);
     this.belowLines += rows;
-    process.stderr.write(`\x1b[${this.belowLines}A`);
+    this.stderr.write(`\x1b[${this.belowLines}A`);
   }
 
   stop(finalMessage?: string): void {
@@ -38,19 +41,19 @@ export class Spinner {
       clearInterval(this.timer);
       this.timer = null;
     }
-    if (process.stderr.isTTY) {
-      process.stderr.write("\r\x1b[K");
+    if (this.stderr.isTTY) {
+      this.stderr.write("\r\x1b[K");
       if (finalMessage) {
-        process.stderr.write(finalMessage);
+        this.stderr.write(finalMessage);
       }
       if (this.belowLines > 0) {
-        process.stderr.write(`\x1b[${this.belowLines}B`);
+        this.stderr.write(`\x1b[${this.belowLines}B`);
       }
       if (finalMessage || this.belowLines > 0) {
-        process.stderr.write("\n");
+        this.stderr.write("\n");
       }
     } else if (finalMessage) {
-      process.stderr.write(`${finalMessage}\n`);
+      this.stderr.write(`${finalMessage}\n`);
     }
   }
 
@@ -68,16 +71,23 @@ export class Spinner {
 
   private draw(): void {
     const glyph = FRAMES[this.frame++ % FRAMES.length];
-    process.stderr.write(`\r${dim(glyph)} ${this.message}`);
+    this.stderr.write(`\r${dim(glyph)} ${this.message}`);
   }
 }
 
+interface SpinnerOutput {
+  isTTY?: boolean;
+  columns?: number;
+  write(text: string): unknown;
+}
+
 export async function withSpinner<T>(
   message: string,
   fn: () => Promise<T>,
   options?: { successMessage?: string; failMessage?: string },
+  output: SpinnerOutput = process.stderr,
 ): Promise<T> {
-  const spinner = new Spinner(message);
+  const spinner = new Spinner(message, output);
   spinner.start();
   try {
     const result = await fn();

package/src/tokenizer/bert-wordpiece.ts

@@ -99,9 +99,6 @@ function whitespaceTokenize(text: string): string[] {
 
 function splitOnPunctuation(text: string): string[] {
   const chars = [...text];
-  if (chars.length === 0) {
-    return [];
-  }
 
   const output: string[][] = [];
   let startNewWord = true;
@@ -138,9 +135,6 @@ function wordpieceTokenize(
   prefix: string,
   maxInputCharsPerWord: number,
 ): string[] {
-  if (token.length === 0) {
-    return [];
-  }
   if (token.length > maxInputCharsPerWord) {
     return [unkToken];
   }