@zokizuan/satori-core 1.0.0 → 1.1.0

package/dist/core/context.d.ts

@@ -1,6 +1,6 @@
 import { Splitter } from '../splitter';
 import { Embedding } from '../embedding';
-import { VectorDatabase } from '../vectordb';
+import { VectorDatabase, IndexCompletionMarkerDocument } from '../vectordb';
 import { SemanticSearchResult } from '../types';
 import { FileSynchronizer } from '../sync/synchronizer';
 export interface ContextConfig {
@@ -126,6 +126,11 @@ export declare class Context {
      * @param threshold Similarity threshold
      */
     semanticSearch(codebasePath: string, query: string, topK?: number, threshold?: number, filterExpr?: string): Promise<SemanticSearchResult[]>;
+    private buildSemanticSearchFilterExpr;
+    private queryCompletionMarkerRows;
+    clearIndexCompletionMarker(codebasePath: string): Promise<void>;
+    writeIndexCompletionMarker(codebasePath: string, marker: IndexCompletionMarkerDocument): Promise<void>;
+    getIndexCompletionMarker(codebasePath: string): Promise<IndexCompletionMarkerDocument | null>;
     /**
      * Check if index exists for codebase
      * @param codebasePath Codebase path to check
@@ -230,17 +235,12 @@ export declare class Context {
      */
     private loadIgnorePatterns;
     /**
-     * Find root-level .xxxignore files in the codebase directory.
-     * v1 policy: only root ignore files are loaded (nested .gitignore files are ignored).
+     * Find supported root ignore files in the codebase directory.
+     * v1 policy: only repo-root .satoriignore and .gitignore are loaded.
      * @param codebasePath Path to the codebase
      * @returns Array of ignore file paths
      */
     private findIgnoreFiles;
-    /**
-     * Load global ignore file from ~/.satori/.satoriignore
-     * @returns Array of ignore patterns
-     */
-    private loadGlobalIgnoreFile;
     /**
      * Load ignore patterns from a specific ignore file
      * @param filePath Path to the ignore file

package/dist/core/context.js

@@ -39,6 +39,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Context = void 0;
 const splitter_1 = require("../splitter");
 const embedding_1 = require("../embedding");
+const vectordb_1 = require("../vectordb");
 const env_manager_1 = require("../utils/env-manager");
 const defaults_1 = require("../config/defaults");
 const language_1 = require("../language");
@@ -283,6 +284,22 @@ class Context {
             this.synchronizers.set(collectionName, newSynchronizer);
         }
         const currentSynchronizer = this.synchronizers.get(collectionName);
+        const collectionExists = await this.vectorDatabase.hasCollection(collectionName);
+        if (!collectionExists) {
+            console.warn(`[Context] ⚠️  Collection '${collectionName}' is missing. Rebuilding full index before incremental sync resumes.`);
+            const changedFiles = this.normalizeRelativePathsForCodebase(codebasePath, await this.getCodeFiles(codebasePath));
+            if (changedFiles.length === 0) {
+                progressCallback?.({ phase: 'No files to index', current: 100, total: 100, percentage: 100 });
+                return { added: 0, removed: 0, modified: 0, changedFiles: [] };
+            }
+            await this.indexCodebase(codebasePath, progressCallback);
+            return {
+                added: changedFiles.length,
+                removed: 0,
+                modified: 0,
+                changedFiles
+            };
+        }
         progressCallback?.({ phase: 'Checking for file changes...', current: 0, total: 100, percentage: 0 });
         const { added, removed, modified } = await currentSynchronizer.checkForChanges();
         const totalChanges = added.length + removed.length + modified.length;
@@ -347,6 +364,7 @@ class Context {
         const isHybrid = this.getIsHybrid();
         const searchType = isHybrid === true ? 'hybrid search' : 'semantic search';
         console.log(`[Context] 🔍 Executing ${searchType}: "${query}" in ${codebasePath}`);
+        const effectiveFilterExpr = this.buildSemanticSearchFilterExpr(filterExpr);
         const normalizeBreadcrumbs = (value) => {
             if (!Array.isArray(value)) {
                 return undefined;
@@ -405,7 +423,8 @@ class Context {
                     params: { k: 100 }
                 },
                 limit: topK,
-                filterExpr
+                threshold,
+                filterExpr: effectiveFilterExpr
             });
             console.log(`[Context] 🔍 Raw search results count: ${searchResults.length}`);
             // 4. Convert to semantic search result format
@@ -432,7 +451,7 @@ class Context {
             // 1. Generate query vector
             const queryEmbedding = await this.embedding.embed(query);
             // 2. Search in vector database
-            const searchResults = await this.vectorDatabase.search(collectionName, queryEmbedding.vector, { topK, threshold, filterExpr });
+            const searchResults = await this.vectorDatabase.search(collectionName, queryEmbedding.vector, { topK, threshold, filterExpr: effectiveFilterExpr });
             // 3. Convert to semantic search result format
             const results = searchResults.map(result => ({
                 content: result.document.content,
@@ -450,6 +469,103 @@ class Context {
             return results;
         }
     }
+    buildSemanticSearchFilterExpr(filterExpr) {
+        const markerExclusion = `fileExtension != "${vectordb_1.INDEX_COMPLETION_MARKER_FILE_EXTENSION}"`;
+        if (!filterExpr || filterExpr.trim().length === 0) {
+            return markerExclusion;
+        }
+        return `(${filterExpr}) and (${markerExclusion})`;
+    }
+    async queryCompletionMarkerRows(collectionName) {
+        return this.vectorDatabase.query(collectionName, `id == "${vectordb_1.INDEX_COMPLETION_MARKER_DOC_ID}"`, ['id', 'metadata'], 8);
+    }
+    async clearIndexCompletionMarker(codebasePath) {
+        const collectionName = this.resolveCollectionName(codebasePath);
+        const hasCollection = await this.vectorDatabase.hasCollection(collectionName);
+        if (!hasCollection) {
+            return;
+        }
+        const rows = await this.queryCompletionMarkerRows(collectionName);
+        const markerIds = rows
+            .map((row) => (typeof row.id === 'string' ? row.id : ''))
+            .filter((id) => id.length > 0);
+        if (markerIds.length === 0) {
+            return;
+        }
+        await this.vectorDatabase.delete(collectionName, Array.from(new Set(markerIds)));
+    }
+    async writeIndexCompletionMarker(codebasePath, marker) {
+        const collectionName = this.resolveCollectionName(codebasePath);
+        const hasCollection = await this.vectorDatabase.hasCollection(collectionName);
+        if (!hasCollection) {
+            throw new Error(`Cannot write completion marker: collection '${collectionName}' does not exist.`);
+        }
+        await this.clearIndexCompletionMarker(codebasePath);
+        const vector = new Array(this.embedding.getDimension()).fill(0);
+        const markerDoc = {
+            id: vectordb_1.INDEX_COMPLETION_MARKER_DOC_ID,
+            vector,
+            content: 'satori index completion marker',
+            relativePath: vectordb_1.INDEX_COMPLETION_MARKER_RELATIVE_PATH,
+            startLine: 0,
+            endLine: 0,
+            fileExtension: vectordb_1.INDEX_COMPLETION_MARKER_FILE_EXTENSION,
+            metadata: marker,
+        };
+        if (this.getIsHybrid() === true) {
+            await this.vectorDatabase.insertHybrid(collectionName, [markerDoc]);
+        }
+        else {
+            await this.vectorDatabase.insert(collectionName, [markerDoc]);
+        }
+    }
+    async getIndexCompletionMarker(codebasePath) {
+        const collectionName = this.resolveCollectionName(codebasePath);
+        const hasCollection = await this.vectorDatabase.hasCollection(collectionName);
+        if (!hasCollection) {
+            return null;
+        }
+        const rows = await this.queryCompletionMarkerRows(collectionName);
+        for (const row of rows) {
+            const rawMetadata = row?.metadata;
+            if (typeof rawMetadata !== 'string') {
+                continue;
+            }
+            try {
+                const parsed = JSON.parse(rawMetadata);
+                if (parsed?.kind !== 'satori_index_completion_v1') {
+                    continue;
+                }
+                if (typeof parsed.codebasePath !== 'string' || typeof parsed.runId !== 'string') {
+                    continue;
+                }
+                if (!parsed.fingerprint || typeof parsed.fingerprint !== 'object') {
+                    continue;
+                }
+                const indexedFiles = Number(parsed.indexedFiles);
+                const totalChunks = Number(parsed.totalChunks);
+                if (!Number.isFinite(indexedFiles) || !Number.isFinite(totalChunks)) {
+                    continue;
+                }
+                if (typeof parsed.completedAt !== 'string' || Number.isNaN(Date.parse(parsed.completedAt))) {
+                    continue;
+                }
+                return {
+                    kind: 'satori_index_completion_v1',
+                    codebasePath: parsed.codebasePath,
+                    fingerprint: parsed.fingerprint,
+                    indexedFiles,
+                    totalChunks,
+                    completedAt: parsed.completedAt,
+                    runId: parsed.runId,
+                };
+            }
+            catch {
+                continue;
+            }
+        }
+        return null;
+    }
     /**
      * Check if index exists for codebase
      * @param codebasePath Codebase path to check
@@ -475,6 +591,8 @@ class Context {
         }
         // Delete snapshot file
         await synchronizer_1.FileSynchronizer.deleteSnapshot(codebasePath);
+        this.synchronizers.delete(collectionName);
+        this.ignoreStateByCollection.delete(collectionName);
         progressCallback?.({ phase: 'Index cleared', current: 100, total: 100, percentage: 100 });
         console.log('[Context] ✅ Index data cleaned');
     }
@@ -722,6 +840,7 @@ class Context {
         let processedFiles = 0;
         let totalChunks = 0;
         let limitReached = false;
+        const describeError = (error) => error instanceof Error ? error.message : String(error);
         for (let i = 0; i < filePaths.length; i++) {
             const filePath = filePaths[i];
             try {
@@ -750,6 +869,7 @@ class Context {
                             if (error instanceof Error) {
                                 console.error('[Context] Stack trace:', error.stack);
                             }
+                            throw new Error(`Failed to persist ${searchType} chunks while indexing ${filePath}: ${describeError(error)}`);
                         }
                         finally {
                             chunkBuffer = []; // Always clear buffer, even on failure
@@ -769,7 +889,8 @@ class Context {
                 }
             }
             catch (error) {
-                console.warn(`[Context] ⚠️  Skipping file ${filePath}: ${error}`);
+                console.error(`[Context] ❌ Failed to index file ${filePath}: ${describeError(error)}`);
+                throw error;
             }
         }
         // Process any remaining chunks in the buffer
@@ -784,6 +905,7 @@ class Context {
                 if (error instanceof Error) {
                     console.error('[Context] Stack trace:', error.stack);
                 }
+                throw new Error(`Failed to persist final ${searchType} chunk batch: ${describeError(error)}`);
             }
         }
         return {
@@ -920,18 +1042,15 @@ class Context {
     async loadIgnorePatterns(codebasePath) {
         try {
             let fileBasedPatterns = [];
-            // Load all .xxxignore files in codebase directory (excluding legacy .contextignore)
+            // v1 policy: only repo-root .satoriignore and .gitignore are supported.
             const ignoreFiles = await this.findIgnoreFiles(codebasePath);
             for (const ignoreFile of ignoreFiles) {
                 const patterns = await this.loadIgnoreFile(ignoreFile, path.basename(ignoreFile));
                 fileBasedPatterns.push(...patterns);
             }
-            // Load global ~/.satori/.satoriignore
-            const globalIgnorePatterns = await this.loadGlobalIgnoreFile();
-            fileBasedPatterns.push(...globalIgnorePatterns);
             this.setFileBasedPatternsForCodebase(codebasePath, fileBasedPatterns);
             if (fileBasedPatterns.length > 0) {
-                console.log(`[Context] 🚫 Loaded total ${fileBasedPatterns.length} ignore patterns from all ignore files`);
+                console.log(`[Context] 🚫 Loaded total ${fileBasedPatterns.length} ignore patterns from supported root ignore files`);
             }
             else {
                 console.log('📄 No ignore files found; effective rules reset to base + runtime custom');
@@ -943,21 +1062,25 @@ class Context {
         }
     }
     /**
-     * Find root-level .xxxignore files in the codebase directory.
-     * v1 policy: only root ignore files are loaded (nested .gitignore files are ignored).
+     * Find supported root ignore files in the codebase directory.
+     * v1 policy: only repo-root .satoriignore and .gitignore are loaded.
      * @param codebasePath Path to the codebase
      * @returns Array of ignore file paths
      */
     async findIgnoreFiles(codebasePath) {
         try {
-            const entries = await fs.promises.readdir(codebasePath, { withFileTypes: true });
             const ignoreFiles = [];
-            for (const entry of entries) {
-                if (entry.isFile() &&
-                    entry.name.startsWith('.') &&
-                    entry.name.endsWith('ignore') &&
-                    entry.name !== '.contextignore') {
-                    ignoreFiles.push(path.join(codebasePath, entry.name));
+            const supportedIgnoreFiles = ['.satoriignore', '.gitignore'];
+            for (const fileName of supportedIgnoreFiles) {
+                const absolutePath = path.join(codebasePath, fileName);
+                try {
+                    const stat = await fs.promises.stat(absolutePath);
+                    if (stat.isFile()) {
+                        ignoreFiles.push(absolutePath);
+                    }
+                }
+                catch {
+                    // Missing ignore file is expected.
                 }
             }
             if (ignoreFiles.length > 0) {
@@ -970,21 +1093,6 @@ class Context {
             return [];
         }
     }
-    /**
-     * Load global ignore file from ~/.satori/.satoriignore
-     * @returns Array of ignore patterns
-     */
-    async loadGlobalIgnoreFile() {
-        try {
-            const homeDir = require('os').homedir();
-            const globalIgnorePath = path.join(homeDir, '.satori', '.satoriignore');
-            return await this.loadIgnoreFile(globalIgnorePath, 'global .satoriignore');
-        }
-        catch (error) {
-            // Global ignore file is optional, don't log warnings
-            return [];
-        }
-    }
     /**
      * Load ignore patterns from a specific ignore file
      * @param filePath Path to the ignore file

package/dist/vectordb/index.d.ts

@@ -1,4 +1,4 @@
-export { VectorDocument, SearchOptions, VectorSearchResult, VectorDatabase, CollectionDetails, VectorStoreBackendInfo, HybridSearchRequest, HybridSearchOptions, HybridSearchResult, RerankStrategy, COLLECTION_LIMIT_MESSAGE } from './types';
+export { VectorDocument, SearchOptions, VectorSearchResult, VectorDatabase, CollectionDetails, VectorStoreBackendInfo, HybridSearchRequest, HybridSearchOptions, HybridSearchResult, RerankStrategy, IndexCompletionFingerprint, IndexCompletionMarkerDocument, INDEX_COMPLETION_MARKER_DOC_ID, INDEX_COMPLETION_MARKER_FILE_EXTENSION, INDEX_COMPLETION_MARKER_RELATIVE_PATH, COLLECTION_LIMIT_MESSAGE } from './types';
 export { MilvusRestfulVectorDatabase, MilvusRestfulConfig } from './milvus-restful-vectordb';
 export { MilvusVectorDatabase, MilvusConfig } from './milvus-vectordb';
 export { ClusterManager, ZillizConfig, Project, Cluster, CreateFreeClusterRequest, CreateFreeClusterResponse, CreateFreeClusterWithDetailsResponse, DescribeClusterResponse } from './zilliz-utils';

package/dist/vectordb/index.js

@@ -1,8 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ClusterManager = exports.MilvusVectorDatabase = exports.MilvusRestfulVectorDatabase = exports.COLLECTION_LIMIT_MESSAGE = void 0;
+exports.ClusterManager = exports.MilvusVectorDatabase = exports.MilvusRestfulVectorDatabase = exports.COLLECTION_LIMIT_MESSAGE = exports.INDEX_COMPLETION_MARKER_RELATIVE_PATH = exports.INDEX_COMPLETION_MARKER_FILE_EXTENSION = exports.INDEX_COMPLETION_MARKER_DOC_ID = void 0;
 // Re-export types and interfaces
 var types_1 = require("./types");
+Object.defineProperty(exports, "INDEX_COMPLETION_MARKER_DOC_ID", { enumerable: true, get: function () { return types_1.INDEX_COMPLETION_MARKER_DOC_ID; } });
+Object.defineProperty(exports, "INDEX_COMPLETION_MARKER_FILE_EXTENSION", { enumerable: true, get: function () { return types_1.INDEX_COMPLETION_MARKER_FILE_EXTENSION; } });
+Object.defineProperty(exports, "INDEX_COMPLETION_MARKER_RELATIVE_PATH", { enumerable: true, get: function () { return types_1.INDEX_COMPLETION_MARKER_RELATIVE_PATH; } });
 Object.defineProperty(exports, "COLLECTION_LIMIT_MESSAGE", { enumerable: true, get: function () { return types_1.COLLECTION_LIMIT_MESSAGE; } });
 // Implementation class exports
 var milvus_restful_vectordb_1 = require("./milvus-restful-vectordb");

package/dist/vectordb/milvus-restful-vectordb.d.ts

@@ -69,11 +69,6 @@ export declare class MilvusRestfulVectorDatabase implements VectorDatabase {
     private createHybridIndexes;
     insertHybrid(collectionName: string, documents: VectorDocument[]): Promise<void>;
     hybridSearch(collectionName: string, searchRequests: HybridSearchRequest[], options?: HybridSearchOptions): Promise<HybridSearchResult[]>;
-    /**
-     * Check collection limit
-     * Returns true if collection can be created, false if limit exceeded
-     * TODO: Implement proper collection limit checking for REST API
-     */
     checkCollectionLimit(): Promise<boolean>;
 }
 //# sourceMappingURL=milvus-restful-vectordb.d.ts.map

package/dist/vectordb/milvus-restful-vectordb.js

@@ -444,7 +444,7 @@ class MilvusRestfulVectorDatabase {
                     score: item.distance || 0
                 };
             });
-            return results;
+            return results.filter((result) => options?.threshold === undefined || result.score >= options.threshold);
         }
         catch (error) {
             console.error(`[MilvusRestfulDB] ❌ Failed to search in collection '${collectionName}':`, error);
@@ -720,7 +720,8 @@ class MilvusRestfulVectorDatabase {
             const results = response.data || [];
             console.log(`[MilvusRestfulDB] ✅ Found ${results.length} results from hybrid search`);
             // Transform response to HybridSearchResult format
-            return results.map((result) => ({
+            return results
+                .map((result) => ({
                 document: {
                     id: result.id,
                     content: result.content,
@@ -733,23 +734,54 @@ class MilvusRestfulVectorDatabase {
                     metadata: JSON.parse(result.metadata || '{}'),
                 },
                 score: result.score || result.distance || 0,
-            }));
+            }))
+                .filter((result) => options?.threshold === undefined || result.score >= options.threshold);
         }
         catch (error) {
             console.error(`[MilvusRestfulDB] ❌ Failed to perform hybrid search on collection '${collectionName}':`, error);
             throw error;
         }
     }
-    /**
-     * Check collection limit
-     * Returns true if collection can be created, false if limit exceeded
-     * TODO: Implement proper collection limit checking for REST API
-     */
     async checkCollectionLimit() {
-        // TODO: Implement REST API version of collection limit checking
-        // For now, always return true to maintain compatibility
-        console.warn('[MilvusRestfulDB] ⚠️  checkCollectionLimit not implemented for REST API - returning true');
-        return true;
+        await this.ensureInitialized();
+        const restfulConfig = this.config;
+        const collectionName = `dummy_collection_${Date.now()}`;
+        const collectionSchema = {
+            collectionName,
+            dbName: restfulConfig.database,
+            schema: {
+                enableDynamicField: false,
+                fields: [
+                    {
+                        fieldName: "id",
+                        dataType: "VarChar",
+                        isPrimary: true,
+                        elementTypeParams: {
+                            max_length: 512
+                        }
+                    },
+                    {
+                        fieldName: "vector",
+                        dataType: "FloatVector",
+                        elementTypeParams: {
+                            dim: 128
+                        }
+                    }
+                ]
+            }
+        };
+        try {
+            await createCollectionWithLimitCheck(this.makeRequest.bind(this), collectionSchema);
+            await this.dropCollection(collectionName);
+            return true;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (message === types_1.COLLECTION_LIMIT_MESSAGE) {
+                return false;
+            }
+            throw error;
+        }
     }
 }
 exports.MilvusRestfulVectorDatabase = MilvusRestfulVectorDatabase;

package/dist/vectordb/milvus-vectordb.js

@@ -442,7 +442,8 @@ class MilvusVectorDatabase {
         if (!searchResult.results || searchResult.results.length === 0) {
             return [];
         }
-        return searchResult.results.map((result) => ({
+        return searchResult.results
+            .map((result) => ({
             document: {
                 id: result.id,
                 vector: queryVector,
@@ -454,7 +455,8 @@ class MilvusVectorDatabase {
                 metadata: JSON.parse(result.metadata || '{}'),
             },
             score: result.score,
-        }));
+        }))
+            .filter((result) => options?.threshold === undefined || result.score >= options.threshold);
     }
     async delete(collectionName, ids) {
         await this.ensureInitialized();
@@ -700,7 +702,8 @@ class MilvusVectorDatabase {
             }
             console.log(`[MilvusDB] ✅ Found ${searchResult.results.length} results from hybrid search`);
             // Transform results to HybridSearchResult format
-            return searchResult.results.map((result) => ({
+            return searchResult.results
+                .map((result) => ({
                 document: {
                     id: result.id,
                     content: result.content,
@@ -713,7 +716,8 @@ class MilvusVectorDatabase {
                     metadata: JSON.parse(result.metadata || '{}'),
                 },
                 score: result.score,
-            }));
+            }))
+                .filter((result) => options?.threshold === undefined || result.score >= options.threshold);
         }
         catch (error) {
             console.error(`[MilvusDB] ❌ Failed to perform hybrid search on collection '${collectionName}':`, error);

package/dist/vectordb/types.d.ts

@@ -23,6 +23,7 @@ export interface HybridSearchRequest {
 export interface HybridSearchOptions {
     rerank?: RerankStrategy;
     limit?: number;
+    threshold?: number;
     filterExpr?: string;
 }
 export interface RerankStrategy {
@@ -46,6 +47,25 @@ export interface VectorStoreBackendInfo {
     transport: 'grpc' | 'rest';
     address?: string;
 }
+export interface IndexCompletionFingerprint {
+    embeddingProvider: string;
+    embeddingModel: string;
+    embeddingDimension: number;
+    vectorStoreProvider: string;
+    schemaVersion: string;
+}
+export interface IndexCompletionMarkerDocument {
+    kind: 'satori_index_completion_v1';
+    codebasePath: string;
+    fingerprint: IndexCompletionFingerprint;
+    indexedFiles: number;
+    totalChunks: number;
+    completedAt: string;
+    runId: string;
+}
+export declare const INDEX_COMPLETION_MARKER_DOC_ID = "__satori_index_completion_marker_v1__";
+export declare const INDEX_COMPLETION_MARKER_FILE_EXTENSION = ".satori_meta";
+export declare const INDEX_COMPLETION_MARKER_RELATIVE_PATH = ".__satori__/index_completion_marker.json";
 export interface VectorDatabase {
     /**
      * Create collection

package/dist/vectordb/types.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.COLLECTION_LIMIT_MESSAGE = void 0;
+exports.COLLECTION_LIMIT_MESSAGE = exports.INDEX_COMPLETION_MARKER_RELATIVE_PATH = exports.INDEX_COMPLETION_MARKER_FILE_EXTENSION = exports.INDEX_COMPLETION_MARKER_DOC_ID = void 0;
+exports.INDEX_COMPLETION_MARKER_DOC_ID = '__satori_index_completion_marker_v1__';
+exports.INDEX_COMPLETION_MARKER_FILE_EXTENSION = '.satori_meta';
+exports.INDEX_COMPLETION_MARKER_RELATIVE_PATH = '.__satori__/index_completion_marker.json';
 /**
  * Special error message for collection limit exceeded
  * This allows us to distinguish it from other errors across all Milvus implementations

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zokizuan/satori-core",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Core semantic indexing engine for Satori's insight-first retrieval",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",