@neuralsea/workspace-indexer 0.1.0 → 0.2.0

package/README.md

@@ -1,4 +1,4 @@
-# @petri-ai/workspace-indexer
+# @neuralsea/workspace-indexer
 
 A **local-first**, **multi-repo** workspace indexer for AI agents (e.g. your custom agent “Damocles”).
 
@@ -31,17 +31,21 @@ This package is designed so Damocles can use the same index in different problem
 npm i @neuralsea/workspace-indexer
 ```
 
-Node 18+ recommended.
+Node 18+ required.
 
 ---
 
 ## Quick start (library)
 
 ```ts
-import { WorkspaceIndexer, OllamaEmbeddingsProvider } from "@neuralsea/workspace-indexer";
+import { WorkspaceIndexer, OllamaEmbeddingsProvider, IndexerProgressObservable } from "@neuralsea/workspace-indexer";
 
 const embedder = new OllamaEmbeddingsProvider({ model: "nomic-embed-text" });
-const ix = new WorkspaceIndexer("/path/to/workspace", embedder);
+
+const progress = new IndexerProgressObservable();
+progress.subscribe(e => console.log(e.type, e));
+
+const ix = new WorkspaceIndexer("/path/to/workspace", embedder, { progress });
 
 await ix.indexAll();
 

package/dist/{chunk-QPQCSCBN.js → chunk-Z3BRYSQM.js}

@@ -726,6 +726,41 @@ async function createVectorIndex(vector) {
   return new BruteForceVectorIndex();
 }
 
+// src/progress.ts
+function toHandler(progress) {
+  if (!progress) return null;
+  if (typeof progress === "function") return progress;
+  if (typeof progress.emit === "function") return (e) => progress.emit(e);
+  return null;
+}
+var IndexerProgressObservable = class {
+  handlers = /* @__PURE__ */ new Set();
+  subscribe(handler) {
+    this.handlers.add(handler);
+    return () => this.handlers.delete(handler);
+  }
+  on(type, handler) {
+    const wrapped = (e) => {
+      if (e.type !== type) return;
+      handler(e);
+    };
+    return this.subscribe(wrapped);
+  }
+  emit(event) {
+    for (const h of this.handlers) {
+      try {
+        h(event);
+      } catch {
+      }
+    }
+  }
+};
+function asProgressSink(progress) {
+  const handler = toHandler(progress);
+  if (!handler) return null;
+  return { emit: handler };
+}
+
 // src/indexer/repoIndexer.ts
 import fs6 from "fs";
 import path8 from "path";
@@ -1359,6 +1394,7 @@ var RepoIndexer = class {
     this.embedder = embedder;
     this.repoRoot = path8.resolve(repoRoot);
     this.repoId = repoIdFromRoot(this.repoRoot);
+    this.progress = asProgressSink(config.progress);
     const cacheDir = config.cacheDir ?? defaultCacheDir();
     this.config = {
       cacheDir,
@@ -1423,6 +1459,7 @@ var RepoIndexer = class {
   repoRoot;
   repoId;
   config;
+  progress = asProgressSink();
   store = null;
   vec = null;
   vecFlushTimer = null;
@@ -1434,6 +1471,12 @@ var RepoIndexer = class {
   fileIgnore = null;
   serial = pLimit(1);
   chunkCache = /* @__PURE__ */ new Map();
+  emitProgress(event) {
+    try {
+      this.progress?.emit(event);
+    } catch {
+    }
+  }
   getCommit() {
     return this.currentCommit;
   }
@@ -1557,6 +1600,7 @@ var RepoIndexer = class {
     this.store.setMeta("commit", commit);
     this.store.setMeta("branch", branch);
     this.store.setMeta("embedderId", this.embedder.id);
+    this.emitProgress({ type: "repo/open", repoRoot: this.repoRoot, repoId: this.repoId, commit, branch });
     const dim = this.embedder.dimension ?? this.store.getAnyEmbeddingDimension();
     if (!dim) return;
     const vec = await this.ensureVectorIndex(dim);
@@ -1573,9 +1617,11 @@ var RepoIndexer = class {
     }
   }
   async indexAll() {
+    const startedAt = Date.now();
     await this.openForCurrentHead();
     if (!this.store) throw new Error("RepoStore not initialised");
     const files = (await listWorkingFiles(this.repoRoot)).filter((p) => this.shouldIndexPath(p));
+    this.emitProgress({ type: "repo/index/start", repoRoot: this.repoRoot, fileCount: files.length });
     const currentSet = new Set(files);
     for (const known of this.store.listAllFiles()) {
       if (!currentSet.has(known)) {
@@ -1585,29 +1631,53 @@ var RepoIndexer = class {
     const limit = pLimit(this.config.embed.concurrency);
     await Promise.all(files.map((f) => limit(() => this.indexFile(f))));
     if (this.vec) await this.flushVectorNow();
+    this.emitProgress({
+      type: "repo/index/done",
+      repoRoot: this.repoRoot,
+      fileCount: files.length,
+      ms: Date.now() - startedAt
+    });
   }
   async indexFile(posixRelPath) {
     return this.serial(async () => {
+      const startedAt = Date.now();
       await this.openForCurrentHead();
       if (!this.store) throw new Error("RepoStore not initialised");
-      if (!this.shouldIndexPath(posixRelPath)) return;
+      if (!this.shouldIndexPath(posixRelPath)) {
+        this.emitProgress({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "filtered" });
+        return;
+      }
+      this.emitProgress({ type: "repo/index/file/start", repoRoot: this.repoRoot, path: posixRelPath });
       const abs = path8.join(this.repoRoot, fromPosixPath(posixRelPath));
       let stat;
       try {
         stat = fs6.statSync(abs);
       } catch {
+        this.emitProgress({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "missing" });
         this.deleteFile(posixRelPath);
         return;
       }
-      if (!stat.isFile()) return;
-      if (stat.size > this.config.maxFileBytes) return;
+      if (!stat.isFile()) {
+        this.emitProgress({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "not_a_file" });
+        return;
+      }
+      if (stat.size > this.config.maxFileBytes) {
+        this.emitProgress({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "too_large" });
+        return;
+      }
       const buf = fs6.readFileSync(abs);
-      if (looksBinary(buf)) return;
+      if (looksBinary(buf)) {
+        this.emitProgress({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "binary" });
+        return;
+      }
       const raw = buf.toString("utf8");
       const redacted = this.applyRedactions(raw);
       const fileHash = sha256Hex(redacted);
       const prev = this.store.getFileHash(posixRelPath);
-      if (prev === fileHash) return;
+      if (prev === fileHash) {
+        this.emitProgress({ type: "repo/index/file/skip", repoRoot: this.repoRoot, path: posixRelPath, reason: "unchanged" });
+        return;
+      }
       const { language, chunks } = chunkSource(posixRelPath, redacted, this.config.chunk);
       if (language === "typescript" || language === "javascript") {
         const rel = extractTsRelations(posixRelPath, redacted);
@@ -1666,6 +1736,13 @@ ${ch.text}`
       for (let start = 0; start < embedTexts.length; start += batchSize) {
         const end = Math.min(embedTexts.length, start + batchSize);
         const batch = embedTexts.slice(start, end);
+        this.emitProgress({
+          type: "repo/index/embed/batch",
+          repoRoot: this.repoRoot,
+          batchSize: batch.length,
+          batchIndex: Math.floor(start / batchSize),
+          batchesTotal: Math.ceil(embedTexts.length / batchSize)
+        });
         const vecs = await this.embedder.embed(batch);
         for (let j = 0; j < vecs.length; j++) {
           const plan = embedPlan[start + j];
@@ -1705,11 +1782,22 @@ ${ch.text}`
       if (dim) {
         await this.ensureVectorIndex(dim);
         if (this.vec) {
-          await this.vec.remove(oldChunkIds);
+          if (oldChunkIds.length) {
+            this.emitProgress({ type: "repo/index/vector/remove", repoRoot: this.repoRoot, ids: oldChunkIds.length });
+            await this.vec.remove(oldChunkIds);
+          }
+          this.emitProgress({ type: "repo/index/vector/upsert", repoRoot: this.repoRoot, points: points.length });
           await this.vec.upsert(points);
           this.scheduleVectorFlush();
         }
       }
+      this.emitProgress({
+        type: "repo/index/file/done",
+        repoRoot: this.repoRoot,
+        path: posixRelPath,
+        chunkCount: combined.length,
+        ms: Date.now() - startedAt
+      });
     });
   }
   async deleteFileInner(posixRelPath) {
@@ -2064,16 +2152,33 @@ var WorkspaceIndexer = class {
     this.embedder = embedder;
     this.config = { ...config };
     if (!this.config.cacheDir) this.config.cacheDir = defaultCacheDir();
+    this.progress = asProgressSink(this.config.progress);
   }
   repos = [];
   config;
+  progress = asProgressSink();
+  emitProgress(event) {
+    try {
+      this.progress?.emit(event);
+    } catch {
+    }
+  }
   async open() {
     const repoRoots = findGitRepos(this.workspaceRoot);
+    this.emitProgress({ type: "workspace/open", workspaceRoot: this.workspaceRoot, repoRoots });
     this.repos = repoRoots.map((r) => new RepoIndexer(r, this.embedder, this.config));
   }
   async indexAll() {
     if (this.repos.length === 0) await this.open();
+    const startedAt = Date.now();
+    this.emitProgress({ type: "workspace/index/start", workspaceRoot: this.workspaceRoot, repoCount: this.repos.length });
     await Promise.all(this.repos.map((r) => r.indexAll()));
+    this.emitProgress({
+      type: "workspace/index/done",
+      workspaceRoot: this.workspaceRoot,
+      repoCount: this.repos.length,
+      ms: Date.now() - startedAt
+    });
   }
   async watch() {
     if (this.repos.length === 0) await this.open();
@@ -2363,6 +2468,8 @@ function loadConfigFile(filePath) {
 
 export {
   createVectorIndex,
+  IndexerProgressObservable,
+  asProgressSink,
   RepoIndexer,
   DEFAULT_PROFILES,
   deepMergeProfile,

package/dist/cli.js

@@ -5,7 +5,7 @@ import {
   OpenAIEmbeddingsProvider,
   WorkspaceIndexer,
   loadConfigFile
-} from "./chunk-QPQCSCBN.js";
+} from "./chunk-Z3BRYSQM.js";
 
 // src/cli.ts
 import yargs from "yargs";

package/dist/index.d.ts

@@ -100,6 +100,75 @@ interface VectorConfig {
     faiss?: FaissVectorConfig;
     custom?: CustomVectorConfig;
 }
+type IndexerProgressEvent = {
+    type: "workspace/open";
+    workspaceRoot: string;
+    repoRoots: string[];
+} | {
+    type: "workspace/index/start";
+    workspaceRoot: string;
+    repoCount: number;
+} | {
+    type: "workspace/index/done";
+    workspaceRoot: string;
+    repoCount: number;
+    ms: number;
+} | {
+    type: "repo/open";
+    repoRoot: string;
+    repoId: string;
+    commit: string;
+    branch: string;
+} | {
+    type: "repo/index/start";
+    repoRoot: string;
+    fileCount: number;
+} | {
+    type: "repo/index/file/start";
+    repoRoot: string;
+    path: string;
+} | {
+    type: "repo/index/file/skip";
+    repoRoot: string;
+    path: string;
+    reason: string;
+} | {
+    type: "repo/index/file/done";
+    repoRoot: string;
+    path: string;
+    chunkCount: number;
+    ms: number;
+} | {
+    type: "repo/index/embed/batch";
+    repoRoot: string;
+    batchSize: number;
+    batchIndex: number;
+    batchesTotal: number;
+} | {
+    type: "repo/index/vector/upsert";
+    repoRoot: string;
+    points: number;
+} | {
+    type: "repo/index/vector/remove";
+    repoRoot: string;
+    ids: number;
+} | {
+    type: "repo/index/done";
+    repoRoot: string;
+    fileCount: number;
+    ms: number;
+} | {
+    type: "error";
+    scope: "workspace" | "repo";
+    message: string;
+    repoRoot?: string;
+    stack?: string;
+};
+type IndexerProgressHandler = (event: IndexerProgressEvent) => void;
+interface IndexerProgressSink {
+    emit(event: IndexerProgressEvent): void;
+}
+type IndexerProgress = IndexerProgressHandler | IndexerProgressSink;
 interface IndexerConfig extends ProfilesConfig {
     cacheDir?: string;
     /** Vector index backend (ANN) configuration. */
@@ -123,6 +192,8 @@ interface IndexerConfig extends ProfilesConfig {
     watch?: {
         debounceMs?: number;
     };
+    /** Optional progress sink (callback or observable). */
+    progress?: IndexerProgress;
     /** Extra ignore files (in addition to git’s excludes). */
     ignoreFiles?: string[];
     /**
@@ -345,6 +416,17 @@ interface VectorIndex {
  */
 declare function createVectorIndex(vector?: VectorConfig): Promise<VectorIndex>;
 
+type Unsubscribe = () => void;
+declare class IndexerProgressObservable implements IndexerProgressSink {
+    private handlers;
+    subscribe(handler: (e: IndexerProgressEvent) => void): Unsubscribe;
+    on<TType extends IndexerProgressEvent["type"]>(type: TType, handler: (e: Extract<IndexerProgressEvent, {
+        type: TType;
+    }>) => void): Unsubscribe;
+    emit(event: IndexerProgressEvent): void;
+}
+declare function asProgressSink(progress?: IndexerProgress): IndexerProgressSink | null;
+
 interface StoredChunkRow {
     id: string;
     path: string;
@@ -423,6 +505,7 @@ declare class RepoIndexer {
     readonly repoRoot: string;
     readonly repoId: string;
     private config;
+    private progress;
     private store;
     private vec;
     private vecFlushTimer;
@@ -435,6 +518,7 @@ declare class RepoIndexer {
     private serial;
     private chunkCache;
     constructor(repoRoot: string, embedder: EmbeddingsProvider, config?: IndexerConfig);
+    private emitProgress;
     getCommit(): string | null;
     getBranch(): string | null;
     getStore(): RepoStore | null;
@@ -488,7 +572,9 @@ declare class WorkspaceIndexer {
     private readonly embedder;
     private repos;
     private config;
+    private progress;
     constructor(workspaceRoot: string, embedder: EmbeddingsProvider, config?: IndexerConfig);
+    private emitProgress;
     open(): Promise<void>;
     indexAll(): Promise<void>;
     watch(): Promise<void>;
@@ -502,4 +588,4 @@ declare class WorkspaceIndexer {
 
 declare function loadConfigFile(filePath: string): IndexerConfig;
 
-export { type ChunkRecord, type ContextBundle, type CustomVectorConfig, DEFAULT_PROFILES, type EmbeddingsProvider, type FaissVectorConfig, HashEmbeddingsProvider, type HnswlibVectorConfig, type IndexerConfig, OllamaEmbeddingsProvider, OpenAIEmbeddingsProvider, type ProfilesConfig, type QdrantVectorConfig, type RelatedContextOptions, type RepoId, RepoIndexer, type RetrievalProfile, type RetrievalProfileName, type RetrievalScope, type RetrievalWeights, type RetrieveOptions, type SearchHit, type SearchOptions, type VectorConfig, type VectorIndex, type VectorIndexInit, type VectorMetric, type VectorPoint, type VectorProviderKind, type VectorSearchHit, WorkspaceIndexer, createVectorIndex, deepMergeProfile, loadConfigFile };
+export { type ChunkRecord, type ContextBundle, type CustomVectorConfig, DEFAULT_PROFILES, type EmbeddingsProvider, type FaissVectorConfig, HashEmbeddingsProvider, type HnswlibVectorConfig, type IndexerConfig, type IndexerProgress, type IndexerProgressEvent, type IndexerProgressHandler, IndexerProgressObservable, type IndexerProgressSink, OllamaEmbeddingsProvider, OpenAIEmbeddingsProvider, type ProfilesConfig, type QdrantVectorConfig, type RelatedContextOptions, type RepoId, RepoIndexer, type RetrievalProfile, type RetrievalProfileName, type RetrievalScope, type RetrievalWeights, type RetrieveOptions, type SearchHit, type SearchOptions, type VectorConfig, type VectorIndex, type VectorIndexInit, type VectorMetric, type VectorPoint, type VectorProviderKind, type VectorSearchHit, WorkspaceIndexer, asProgressSink, createVectorIndex, deepMergeProfile, loadConfigFile };

package/dist/index.js

@@ -1,21 +1,25 @@
 import {
   DEFAULT_PROFILES,
   HashEmbeddingsProvider,
+  IndexerProgressObservable,
   OllamaEmbeddingsProvider,
   OpenAIEmbeddingsProvider,
   RepoIndexer,
   WorkspaceIndexer,
+  asProgressSink,
   createVectorIndex,
   deepMergeProfile,
   loadConfigFile
-} from "./chunk-QPQCSCBN.js";
+} from "./chunk-Z3BRYSQM.js";
 export {
   DEFAULT_PROFILES,
   HashEmbeddingsProvider,
+  IndexerProgressObservable,
   OllamaEmbeddingsProvider,
   OpenAIEmbeddingsProvider,
   RepoIndexer,
   WorkspaceIndexer,
+  asProgressSink,
   createVectorIndex,
   deepMergeProfile,
   loadConfigFile

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neuralsea/workspace-indexer",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Local-first multi-repo workspace indexer (semantic embeddings + git-aware incremental updates + hybrid retrieval profiles) for AI agents.",
   "type": "module",
   "main": "./dist/index.js",