@zokizuan/satori-core 0.2.0 → 1.1.0

package/README.md

@@ -11,7 +11,7 @@ Maintained by: `ham-zax` (`@zokizuan`).
 - code splitting (AST + LangChain fallback)
 - embedding generation
 - vector persistence and search via Milvus
-- incremental sync via Merkle-based change detection
+- incremental sync via stat-first change detection with hash-on-change verification
 
 ## Install
 

package/dist/core/context.d.ts

@@ -1,6 +1,6 @@
 import { Splitter } from '../splitter';
 import { Embedding } from '../embedding';
-import { VectorDatabase } from '../vectordb';
+import { VectorDatabase, IndexCompletionMarkerDocument } from '../vectordb';
 import { SemanticSearchResult } from '../types';
 import { FileSynchronizer } from '../sync/synchronizer';
 export interface ContextConfig {
@@ -126,6 +126,11 @@ export declare class Context {
      * @param threshold Similarity threshold
      */
     semanticSearch(codebasePath: string, query: string, topK?: number, threshold?: number, filterExpr?: string): Promise<SemanticSearchResult[]>;
+    private buildSemanticSearchFilterExpr;
+    private queryCompletionMarkerRows;
+    clearIndexCompletionMarker(codebasePath: string): Promise<void>;
+    writeIndexCompletionMarker(codebasePath: string, marker: IndexCompletionMarkerDocument): Promise<void>;
+    getIndexCompletionMarker(codebasePath: string): Promise<IndexCompletionMarkerDocument | null>;
     /**
      * Check if index exists for codebase
      * @param codebasePath Codebase path to check
@@ -230,17 +235,12 @@ export declare class Context {
      */
     private loadIgnorePatterns;
     /**
-     * Find root-level .xxxignore files in the codebase directory.
-     * v1 policy: only root ignore files are loaded (nested .gitignore files are ignored).
+     * Find supported root ignore files in the codebase directory.
+     * v1 policy: only repo-root .satoriignore and .gitignore are loaded.
      * @param codebasePath Path to the codebase
      * @returns Array of ignore file paths
      */
     private findIgnoreFiles;
-    /**
-     * Load global ignore file from ~/.satori/.satoriignore
-     * @returns Array of ignore patterns
-     */
-    private loadGlobalIgnoreFile;
     /**
      * Load ignore patterns from a specific ignore file
      * @param filePath Path to the ignore file

package/dist/core/context.js

@@ -39,6 +39,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Context = void 0;
 const splitter_1 = require("../splitter");
 const embedding_1 = require("../embedding");
+const vectordb_1 = require("../vectordb");
 const env_manager_1 = require("../utils/env-manager");
 const defaults_1 = require("../config/defaults");
 const language_1 = require("../language");
@@ -283,6 +284,22 @@ class Context {
             this.synchronizers.set(collectionName, newSynchronizer);
         }
         const currentSynchronizer = this.synchronizers.get(collectionName);
+        const collectionExists = await this.vectorDatabase.hasCollection(collectionName);
+        if (!collectionExists) {
+            console.warn(`[Context] ⚠️  Collection '${collectionName}' is missing. Rebuilding full index before incremental sync resumes.`);
+            const changedFiles = this.normalizeRelativePathsForCodebase(codebasePath, await this.getCodeFiles(codebasePath));
+            if (changedFiles.length === 0) {
+                progressCallback?.({ phase: 'No files to index', current: 100, total: 100, percentage: 100 });
+                return { added: 0, removed: 0, modified: 0, changedFiles: [] };
+            }
+            await this.indexCodebase(codebasePath, progressCallback);
+            return {
+                added: changedFiles.length,
+                removed: 0,
+                modified: 0,
+                changedFiles
+            };
+        }
         progressCallback?.({ phase: 'Checking for file changes...', current: 0, total: 100, percentage: 0 });
         const { added, removed, modified } = await currentSynchronizer.checkForChanges();
         const totalChanges = added.length + removed.length + modified.length;
@@ -347,6 +364,7 @@ class Context {
         const isHybrid = this.getIsHybrid();
         const searchType = isHybrid === true ? 'hybrid search' : 'semantic search';
         console.log(`[Context] 🔍 Executing ${searchType}: "${query}" in ${codebasePath}`);
+        const effectiveFilterExpr = this.buildSemanticSearchFilterExpr(filterExpr);
         const normalizeBreadcrumbs = (value) => {
             if (!Array.isArray(value)) {
                 return undefined;
@@ -405,7 +423,8 @@ class Context {
                     params: { k: 100 }
                 },
                 limit: topK,
-                filterExpr
+                threshold,
+                filterExpr: effectiveFilterExpr
             });
             console.log(`[Context] 🔍 Raw search results count: ${searchResults.length}`);
             // 4. Convert to semantic search result format
@@ -432,7 +451,7 @@ class Context {
             // 1. Generate query vector
             const queryEmbedding = await this.embedding.embed(query);
             // 2. Search in vector database
-            const searchResults = await this.vectorDatabase.search(collectionName, queryEmbedding.vector, { topK, threshold, filterExpr });
+            const searchResults = await this.vectorDatabase.search(collectionName, queryEmbedding.vector, { topK, threshold, filterExpr: effectiveFilterExpr });
             // 3. Convert to semantic search result format
             const results = searchResults.map(result => ({
                 content: result.document.content,
@@ -450,6 +469,103 @@ class Context {
             return results;
         }
     }
+    buildSemanticSearchFilterExpr(filterExpr) {
+        const markerExclusion = `fileExtension != "${vectordb_1.INDEX_COMPLETION_MARKER_FILE_EXTENSION}"`;
+        if (!filterExpr || filterExpr.trim().length === 0) {
+            return markerExclusion;
+        }
+        return `(${filterExpr}) and (${markerExclusion})`;
+    }
+    async queryCompletionMarkerRows(collectionName) {
+        return this.vectorDatabase.query(collectionName, `id == "${vectordb_1.INDEX_COMPLETION_MARKER_DOC_ID}"`, ['id', 'metadata'], 8);
+    }
+    async clearIndexCompletionMarker(codebasePath) {
+        const collectionName = this.resolveCollectionName(codebasePath);
+        const hasCollection = await this.vectorDatabase.hasCollection(collectionName);
+        if (!hasCollection) {
+            return;
+        }
+        const rows = await this.queryCompletionMarkerRows(collectionName);
+        const markerIds = rows
+            .map((row) => (typeof row.id === 'string' ? row.id : ''))
+            .filter((id) => id.length > 0);
+        if (markerIds.length === 0) {
+            return;
+        }
+        await this.vectorDatabase.delete(collectionName, Array.from(new Set(markerIds)));
+    }
+    async writeIndexCompletionMarker(codebasePath, marker) {
+        const collectionName = this.resolveCollectionName(codebasePath);
+        const hasCollection = await this.vectorDatabase.hasCollection(collectionName);
+        if (!hasCollection) {
+            throw new Error(`Cannot write completion marker: collection '${collectionName}' does not exist.`);
+        }
+        await this.clearIndexCompletionMarker(codebasePath);
+        const vector = new Array(this.embedding.getDimension()).fill(0);
+        const markerDoc = {
+            id: vectordb_1.INDEX_COMPLETION_MARKER_DOC_ID,
+            vector,
+            content: 'satori index completion marker',
+            relativePath: vectordb_1.INDEX_COMPLETION_MARKER_RELATIVE_PATH,
+            startLine: 0,
+            endLine: 0,
+            fileExtension: vectordb_1.INDEX_COMPLETION_MARKER_FILE_EXTENSION,
+            metadata: marker,
+        };
+        if (this.getIsHybrid() === true) {
+            await this.vectorDatabase.insertHybrid(collectionName, [markerDoc]);
+        }
+        else {
+            await this.vectorDatabase.insert(collectionName, [markerDoc]);
+        }
+    }
+    async getIndexCompletionMarker(codebasePath) {
+        const collectionName = this.resolveCollectionName(codebasePath);
+        const hasCollection = await this.vectorDatabase.hasCollection(collectionName);
+        if (!hasCollection) {
+            return null;
+        }
+        const rows = await this.queryCompletionMarkerRows(collectionName);
+        for (const row of rows) {
+            const rawMetadata = row?.metadata;
+            if (typeof rawMetadata !== 'string') {
+                continue;
+            }
+            try {
+                const parsed = JSON.parse(rawMetadata);
+                if (parsed?.kind !== 'satori_index_completion_v1') {
+                    continue;
+                }
+                if (typeof parsed.codebasePath !== 'string' || typeof parsed.runId !== 'string') {
+                    continue;
+                }
+                if (!parsed.fingerprint || typeof parsed.fingerprint !== 'object') {
+                    continue;
+                }
+                const indexedFiles = Number(parsed.indexedFiles);
+                const totalChunks = Number(parsed.totalChunks);
+                if (!Number.isFinite(indexedFiles) || !Number.isFinite(totalChunks)) {
+                    continue;
+                }
+                if (typeof parsed.completedAt !== 'string' || Number.isNaN(Date.parse(parsed.completedAt))) {
+                    continue;
+                }
+                return {
+                    kind: 'satori_index_completion_v1',
+                    codebasePath: parsed.codebasePath,
+                    fingerprint: parsed.fingerprint,
+                    indexedFiles,
+                    totalChunks,
+                    completedAt: parsed.completedAt,
+                    runId: parsed.runId,
+                };
+            }
+            catch {
+                continue;
+            }
+        }
+        return null;
+    }
     /**
      * Check if index exists for codebase
      * @param codebasePath Codebase path to check
@@ -475,6 +591,8 @@ class Context {
         }
         // Delete snapshot file
         await synchronizer_1.FileSynchronizer.deleteSnapshot(codebasePath);
+        this.synchronizers.delete(collectionName);
+        this.ignoreStateByCollection.delete(collectionName);
         progressCallback?.({ phase: 'Index cleared', current: 100, total: 100, percentage: 100 });
         console.log('[Context] ✅ Index data cleaned');
     }
@@ -722,6 +840,7 @@ class Context {
         let processedFiles = 0;
         let totalChunks = 0;
         let limitReached = false;
+        const describeError = (error) => error instanceof Error ? error.message : String(error);
         for (let i = 0; i < filePaths.length; i++) {
             const filePath = filePaths[i];
             try {
@@ -750,6 +869,7 @@ class Context {
                             if (error instanceof Error) {
                                 console.error('[Context] Stack trace:', error.stack);
                             }
+                            throw new Error(`Failed to persist ${searchType} chunks while indexing ${filePath}: ${describeError(error)}`);
                         }
                         finally {
                             chunkBuffer = []; // Always clear buffer, even on failure
@@ -769,7 +889,8 @@ class Context {
                 }
             }
             catch (error) {
-                console.warn(`[Context] ⚠️  Skipping file ${filePath}: ${error}`);
+                console.error(`[Context] ❌ Failed to index file ${filePath}: ${describeError(error)}`);
+                throw error;
             }
         }
         // Process any remaining chunks in the buffer
@@ -784,6 +905,7 @@ class Context {
                 if (error instanceof Error) {
                     console.error('[Context] Stack trace:', error.stack);
                 }
+                throw new Error(`Failed to persist final ${searchType} chunk batch: ${describeError(error)}`);
             }
         }
         return {
@@ -920,18 +1042,15 @@ class Context {
     async loadIgnorePatterns(codebasePath) {
         try {
             let fileBasedPatterns = [];
-            // Load all .xxxignore files in codebase directory (excluding legacy .contextignore)
+            // v1 policy: only repo-root .satoriignore and .gitignore are supported.
             const ignoreFiles = await this.findIgnoreFiles(codebasePath);
             for (const ignoreFile of ignoreFiles) {
                 const patterns = await this.loadIgnoreFile(ignoreFile, path.basename(ignoreFile));
                 fileBasedPatterns.push(...patterns);
             }
-            // Load global ~/.satori/.satoriignore
-            const globalIgnorePatterns = await this.loadGlobalIgnoreFile();
-            fileBasedPatterns.push(...globalIgnorePatterns);
             this.setFileBasedPatternsForCodebase(codebasePath, fileBasedPatterns);
             if (fileBasedPatterns.length > 0) {
-                console.log(`[Context] 🚫 Loaded total ${fileBasedPatterns.length} ignore patterns from all ignore files`);
+                console.log(`[Context] 🚫 Loaded total ${fileBasedPatterns.length} ignore patterns from supported root ignore files`);
             }
             else {
                 console.log('📄 No ignore files found; effective rules reset to base + runtime custom');
@@ -943,21 +1062,25 @@ class Context {
         }
     }
     /**
-     * Find root-level .xxxignore files in the codebase directory.
-     * v1 policy: only root ignore files are loaded (nested .gitignore files are ignored).
+     * Find supported root ignore files in the codebase directory.
+     * v1 policy: only repo-root .satoriignore and .gitignore are loaded.
      * @param codebasePath Path to the codebase
      * @returns Array of ignore file paths
      */
     async findIgnoreFiles(codebasePath) {
         try {
-            const entries = await fs.promises.readdir(codebasePath, { withFileTypes: true });
             const ignoreFiles = [];
-            for (const entry of entries) {
-                if (entry.isFile() &&
-                    entry.name.startsWith('.') &&
-                    entry.name.endsWith('ignore') &&
-                    entry.name !== '.contextignore') {
-                    ignoreFiles.push(path.join(codebasePath, entry.name));
+            const supportedIgnoreFiles = ['.satoriignore', '.gitignore'];
+            for (const fileName of supportedIgnoreFiles) {
+                const absolutePath = path.join(codebasePath, fileName);
+                try {
+                    const stat = await fs.promises.stat(absolutePath);
+                    if (stat.isFile()) {
+                        ignoreFiles.push(absolutePath);
+                    }
+                }
+                catch {
+                    // Missing ignore file is expected.
                 }
             }
             if (ignoreFiles.length > 0) {
@@ -970,21 +1093,6 @@ class Context {
             return [];
         }
     }
-    /**
-     * Load global ignore file from ~/.satori/.satoriignore
-     * @returns Array of ignore patterns
-     */
-    async loadGlobalIgnoreFile() {
-        try {
-            const homeDir = require('os').homedir();
-            const globalIgnorePath = path.join(homeDir, '.satori', '.satoriignore');
-            return await this.loadIgnoreFile(globalIgnorePath, 'global .satoriignore');
-        }
-        catch (error) {
-            // Global ignore file is optional, don't log warnings
-            return [];
-        }
-    }
     /**
      * Load ignore patterns from a specific ignore file
      * @param filePath Path to the ignore file

package/dist/sync/merkle.d.ts

@@ -1,26 +1,7 @@
-export interface MerkleDAGNode {
-    id: string;
-    hash: string;
-    data: string;
-    parents: string[];
-    children: string[];
-}
-export declare class MerkleDAG {
-    nodes: Map<string, MerkleDAGNode>;
-    rootIds: string[];
-    constructor();
-    private hash;
-    addNode(data: string, parentId?: string): string;
-    getNode(nodeId: string): MerkleDAGNode | undefined;
-    getAllNodes(): MerkleDAGNode[];
-    getRootNodes(): MerkleDAGNode[];
-    getLeafNodes(): MerkleDAGNode[];
-    serialize(): any;
-    static deserialize(data: any): MerkleDAG;
-    static compare(dag1: MerkleDAG, dag2: MerkleDAG): {
-        added: string[];
-        removed: string[];
-        modified: string[];
-    };
-}
+/**
+ * Compute a deterministic Merkle-like root from file hashes.
+ * Input keys are expected to be normalized relative paths.
+ */
+export declare function computeMerkleRoot(fileHashes: Map<string, string>): string;
+export declare function computeMerkleLeaf(relativePath: string, hash: string): string;
 //# sourceMappingURL=merkle.d.ts.map

package/dist/sync/merkle.js

@@ -33,80 +33,28 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MerkleDAG = void 0;
+exports.computeMerkleRoot = computeMerkleRoot;
+exports.computeMerkleLeaf = computeMerkleLeaf;
 const crypto = __importStar(require("crypto"));
-class MerkleDAG {
-    constructor() {
-        this.nodes = new Map();
-        this.rootIds = [];
-    }
-    hash(data) {
-        return crypto.createHash('sha256').update(data).digest('hex');
-    }
-    addNode(data, parentId) {
-        const nodeId = this.hash(data);
-        const node = {
-            id: nodeId,
-            hash: nodeId,
-            data,
-            parents: [],
-            children: []
-        };
-        // If there's a parent, create the relationship
-        if (parentId) {
-            const parentNode = this.nodes.get(parentId);
-            if (parentNode) {
-                node.parents.push(parentId);
-                parentNode.children.push(nodeId);
-                this.nodes.set(parentId, parentNode);
-            }
-        }
-        else {
-            // If no parent, it's a root node
-            this.rootIds.push(nodeId);
-        }
-        this.nodes.set(nodeId, node);
-        return nodeId;
-    }
-    getNode(nodeId) {
-        return this.nodes.get(nodeId);
-    }
-    getAllNodes() {
-        return Array.from(this.nodes.values());
-    }
-    getRootNodes() {
-        return this.rootIds.map(id => this.nodes.get(id)).filter(Boolean);
-    }
-    getLeafNodes() {
-        return Array.from(this.nodes.values()).filter(node => node.children.length === 0);
-    }
-    serialize() {
-        return {
-            nodes: Array.from(this.nodes.entries()),
-            rootIds: this.rootIds
-        };
-    }
-    static deserialize(data) {
-        const dag = new MerkleDAG();
-        dag.nodes = new Map(data.nodes);
-        dag.rootIds = data.rootIds;
-        return dag;
-    }
-    static compare(dag1, dag2) {
-        const nodes1 = new Map(Array.from(dag1.getAllNodes()).map(n => [n.id, n]));
-        const nodes2 = new Map(Array.from(dag2.getAllNodes()).map(n => [n.id, n]));
-        const added = Array.from(nodes2.keys()).filter(k => !nodes1.has(k));
-        const removed = Array.from(nodes1.keys()).filter(k => !nodes2.has(k));
-        // For modified, we'll check if the data has changed for nodes that exist in both
-        const modified = [];
-        for (const [id, node1] of Array.from(nodes1.entries())) {
-            const node2 = nodes2.get(id);
-            if (node2 && node1.data !== node2.data) {
-                modified.push(id);
-            }
-        }
-        return { added, removed, modified };
-    }
+function hashChunk(value) {
+    return crypto.createHash('sha256').update(value).digest('hex');
+}
+/**
+ * Compute a deterministic Merkle-like root from file hashes.
+ * Input keys are expected to be normalized relative paths.
+ */
+function computeMerkleRoot(fileHashes) {
+    const hasher = crypto.createHash('sha256');
+    const sortedEntries = Array.from(fileHashes.entries()).sort(([a], [b]) => a.localeCompare(b));
+    for (const [relativePath, hash] of sortedEntries) {
+        hasher.update(relativePath);
+        hasher.update('\0');
+        hasher.update(hash);
+        hasher.update('\n');
+    }
+    return hasher.digest('hex');
+}
+function computeMerkleLeaf(relativePath, hash) {
+    return hashChunk(`${relativePath}\0${hash}`);
 }
-exports.MerkleDAG = MerkleDAG;
 //# sourceMappingURL=merkle.js.map

package/dist/sync/synchronizer.d.ts

@@ -1,33 +1,55 @@
+export interface FileChangeResult {
+    added: string[];
+    removed: string[];
+    modified: string[];
+    hashedCount: number;
+    partialScan: boolean;
+    unscannedDirPrefixes: string[];
+    fullHashRun: boolean;
+}
 export declare class FileSynchronizer {
     private fileHashes;
-    private merkleDAG;
+    private fileStats;
+    private merkleRoot;
     private rootDir;
     private snapshotPath;
     private ignorePatterns;
     private ignoreMatcher;
+    private partialScan;
+    private unscannedDirPrefixes;
+    private fullHashCounter;
     constructor(rootDir: string, ignorePatterns?: string[]);
-    private getSnapshotPath;
-    private hashFile;
-    private generateFileHashes;
+    static canonicalizeSnapshotIdentityPath(codebasePath: string): string;
+    static snapshotPathFromCanonicalPath(canonicalPath: string): string;
+    static getSnapshotPathForCodebase(codebasePath: string): string;
+    private static trimTrailingSeparators;
+    private normalizeRelPath;
+    private isPathWithinPrefix;
+    private normalizeAndCompressPrefixes;
     private shouldIgnore;
-    private buildMerkleDAG;
-    initialize(): Promise<void>;
-    checkForChanges(): Promise<{
-        added: string[];
-        removed: string[];
-        modified: string[];
-    }>;
+    private parsePositiveInt;
+    private getHashConcurrency;
+    private getFullHashInterval;
+    private hashFileBytes;
+    private isSignatureEqual;
+    private scanDirectory;
+    private hashCandidatesWithConcurrency;
+    private buildEffectiveState;
     private compareStates;
+    private arraysEqual;
+    private saveSnapshot;
+    private loadSnapshot;
+    private scanCurrentState;
+    initialize(): Promise<void>;
+    checkForChanges(): Promise<FileChangeResult>;
     getFileHash(filePath: string): string | undefined;
     /**
      * Return tracked (currently considered indexable) relative file paths.
      * This reflects the synchronizer snapshot under the active ignore rules.
      */
     getTrackedRelativePaths(): string[];
-    private saveSnapshot;
-    private loadSnapshot;
     /**
-     * Delete snapshot file for a given codebase path
+     * Delete snapshot file for a given codebase path.
      */
     static deleteSnapshot(codebasePath: string): Promise<void>;
 }