@context-os/core 1.0.1 → 1.6.1

package/dist/context.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Discovers the workspace root by looking for root/soul.md in parent directories.
+ */
+export declare function findWorkspaceRoot(): string;
+export declare const workspaceRoot: string;
+export declare function getWorkspaceRoot(): string;
+/**
+ * Standard ContextOS "Buckets" for security isolation.
+ */
+export declare const ALLOWED_BUCKETS: string[];

package/dist/context.js

@@ -0,0 +1,34 @@
+import path from 'node:path';
+import fs from 'node:fs';
+/**
+ * Discovers the workspace root by looking for root/soul.md in parent directories.
+ */
+export function findWorkspaceRoot() {
+    let current = process.cwd();
+    const root = path.parse(current).root;
+    while (current !== root) {
+        if (fs.existsSync(path.join(current, "root", "soul.md"))) {
+            return fs.realpathSync(current);
+        }
+        current = path.dirname(current);
+    }
+    return fs.realpathSync(process.cwd()); // Fallback to CWD
+}
+export const workspaceRoot = findWorkspaceRoot();
+export function getWorkspaceRoot() {
+    return workspaceRoot;
+}
+/**
+ * Standard ContextOS "Buckets" for security isolation.
+ */
+export const ALLOWED_BUCKETS = [
+    "projects",
+    "knowledge",
+    "schemas",
+    "archive",
+    "log",
+    "orgs",
+    "root",
+    "docs",
+    "prompts"
+];

package/dist/index.d.ts

@@ -1,12 +1,11 @@
-/**
- * Discovers the workspace root by looking for root/soul.md in parent directories.
- */
-export declare function findWorkspaceRoot(): string;
-export declare const workspaceRoot: string;
-/**
- * Standard ContextOS "Buckets" for security isolation.
- */
-export declare const ALLOWED_BUCKETS: string[];
+export * from './context.js';
+export * from './indexer.js';
+export * from './services/intelligence.js';
+export * from './services/validation.js';
+export * from './services/workspace.js';
+export * from './services/knowledge-graph.js';
+export * from './services/sampling.js';
+export * from './services/watch.js';
 /**
  * Validates that a path is within the workspace root and inside an allowed bucket.
  */

package/dist/index.js

@@ -1,33 +1,15 @@
 import path from 'node:path';
 import fs from 'node:fs';
 import { spawn } from 'node:child_process';
-/**
- * Discovers the workspace root by looking for root/soul.md in parent directories.
- */
-export function findWorkspaceRoot() {
-    let current = process.cwd();
-    const root = path.parse(current).root;
-    while (current !== root) {
-        if (fs.existsSync(path.join(current, "root", "soul.md"))) {
-            return fs.realpathSync(current);
-        }
-        current = path.dirname(current);
-    }
-    return fs.realpathSync(process.cwd()); // Fallback to CWD
-}
-export const workspaceRoot = findWorkspaceRoot();
-/**
- * Standard ContextOS "Buckets" for security isolation.
- */
-export const ALLOWED_BUCKETS = [
-    "projects",
-    "knowledge",
-    "schemas",
-    "archive",
-    "log",
-    "orgs",
-    "root"
-];
+import { workspaceRoot, ALLOWED_BUCKETS } from './context.js';
+export * from './context.js';
+export * from './indexer.js';
+export * from './services/intelligence.js';
+export * from './services/validation.js';
+export * from './services/workspace.js';
+export * from './services/knowledge-graph.js';
+export * from './services/sampling.js';
+export * from './services/watch.js';
 /**
  * Validates that a path is within the workspace root and inside an allowed bucket.
  */

package/dist/indexer.d.ts

@@ -0,0 +1,47 @@
+export interface IndexRecord {
+    path: string;
+    title: string;
+    tags: string[];
+    status: string;
+    lastModified: number;
+    excerpt: string;
+    content: string;
+    mentions: string[];
+}
+export interface ContextIndex {
+    version: string;
+    lastUpdated: number;
+    records: IndexRecord[];
+    provider?: string;
+}
+export declare class ContextIndexer {
+    private indexPath;
+    private previousIndex;
+    private dbService;
+    private embeddingService;
+    constructor();
+    /**
+     * Recursively indexes the workspace metadata.
+     * Implements Incremental logic: only parses files changed since last index.
+     */
+    reindex(options?: {
+        force?: boolean;
+    }): Promise<ContextIndex>;
+    private scanDirectory;
+    /**
+     * Public method to index or update a single file.
+     * Synchronizes both SQLite and the JSON index.
+     */
+    indexFile(filePath: string, mtimeMs?: number): Promise<IndexRecord | null>;
+    /**
+     * Public method to remove a file from all index layers.
+     */
+    removeFile(relativePath: string): Promise<void>;
+    private updateJsonIndex;
+    private parseMarkdownFile;
+    /**
+     * Searches the local index.
+     */
+    search(query: string): Promise<IndexRecord[]>;
+}
+export declare const globalIndexer: ContextIndexer;

package/dist/indexer.js

@@ -0,0 +1,206 @@
+import path from 'node:path';
+import fs from 'fs-extra';
+import { workspaceRoot, ALLOWED_BUCKETS } from './context.js';
+import { validationService } from './services/validation.js';
+import { DatabaseService } from './services/database.js';
+import { EmbeddingService } from './services/embedding.js';
+export class ContextIndexer {
+    indexPath;
+    previousIndex = null;
+    dbService;
+    embeddingService;
+    constructor() {
+        this.indexPath = path.join(workspaceRoot, '.context-index.json');
+        this.dbService = new DatabaseService(workspaceRoot);
+        // Load API key if present for Elite mode
+        const geminiKey = process.env.GEMINI_API_KEY;
+        this.embeddingService = new EmbeddingService(geminiKey);
+    }
+    /**
+     * Recursively indexes the workspace metadata.
+     * Implements Incremental logic: only parses files changed since last index.
+     */
+    async reindex(options = {}) {
+        // 1. Load previous index for incremental comparison
+        if (!options.force && await fs.pathExists(this.indexPath)) {
+            try {
+                this.previousIndex = await fs.readJSON(this.indexPath);
+                // Upgrade from v1.2.x to v1.3.0 forces a semantic re-index
+                if (this.previousIndex && this.previousIndex.version !== '1.3.0') {
+                    this.previousIndex = null;
+                }
+            }
+            catch (e) {
+                this.previousIndex = null;
+            }
+        }
+        const index = {
+            version: '1.3.0',
+            lastUpdated: Date.now(),
+            records: [],
+            provider: await this.embeddingService.getProviderName()
+        };
+        const existingRecordMap = new Map();
+        if (this.previousIndex) {
+            this.previousIndex.records.forEach(r => existingRecordMap.set(r.path, r));
+        }
+        // 2. Scan buckets incrementally
+        for (const bucket of ALLOWED_BUCKETS) {
+            const bucketPath = path.join(workspaceRoot, bucket);
+            if (await fs.pathExists(bucketPath)) {
+                await this.scanDirectory(bucketPath, index.records, existingRecordMap);
+            }
+        }
+        // 3. Atomic persistence
+        const tempPath = `${this.indexPath}.tmp`;
+        await fs.writeJSON(tempPath, index, { spaces: 2 });
+        await fs.move(tempPath, this.indexPath, { overwrite: true });
+        return index;
+    }
+    async scanDirectory(dir, records, existingRecordMap) {
+        const entries = await fs.readdir(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                await this.scanDirectory(fullPath, records, existingRecordMap);
+                continue;
+            }
+            if (entry.name.endsWith('.md')) {
+                const relativePath = path.relative(workspaceRoot, fullPath);
+                const stats = await fs.stat(fullPath);
+                const existing = existingRecordMap.get(relativePath);
+                // Incremental Check: Skip parsing if mtime matches
+                if (existing && existing.lastModified === stats.mtimeMs) {
+                    records.push(existing);
+                    continue;
+                }
+                const record = await this.indexFile(fullPath, stats.mtimeMs);
+                if (record) {
+                    records.push(record);
+                }
+            }
+        }
+    }
+    /**
+     * Public method to index or update a single file.
+     * Synchronizes both SQLite and the JSON index.
+     */
+    async indexFile(filePath, mtimeMs) {
+        if (!mtimeMs) {
+            const stats = await fs.stat(filePath);
+            mtimeMs = stats.mtimeMs;
+        }
+        const record = await this.parseMarkdownFile(filePath, mtimeMs);
+        if (record) {
+            // 1. SQLite Upsert + Semantic Generation
+            const { id } = this.dbService.upsertDocument({
+                path: record.path,
+                title: record.title,
+                content: record.content,
+                excerpt: record.excerpt,
+                mtime: record.lastModified,
+                metadata: JSON.stringify(record.tags)
+            });
+            // Generate Vector for Semantic Layer
+            const embedding = await this.embeddingService.generate(`${record.title}\n${record.excerpt}\n${record.content}`);
+            this.dbService.upsertVector(id, embedding, await this.embeddingService.getProviderName());
+            // 2. Update JSON Index (if it exists)
+            await this.updateJsonIndex(record);
+        }
+        return record;
+    }
+    /**
+     * Public method to remove a file from all index layers.
+     */
+    async removeFile(relativePath) {
+        // 1. SQLite Cleanup
+        this.dbService.removeDocument(relativePath);
+        // 2. JSON Cleanup
+        if (await fs.pathExists(this.indexPath)) {
+            try {
+                const index = await fs.readJSON(this.indexPath);
+                index.records = index.records.filter(r => r.path !== relativePath);
+                index.lastUpdated = Date.now();
+                await fs.writeJSON(this.indexPath, index, { spaces: 2 });
+            }
+            catch (e) {
+                // Ignore parsing errors
+            }
+        }
+    }
+    async updateJsonIndex(record) {
+        if (!(await fs.pathExists(this.indexPath)))
+            return;
+        try {
+            const index = await fs.readJSON(this.indexPath);
+            const existingIndex = index.records.findIndex(r => r.path === record.path);
+            if (existingIndex >= 0) {
+                index.records[existingIndex] = record;
+            }
+            else {
+                index.records.push(record);
+            }
+            index.lastUpdated = Date.now();
+            await fs.writeJSON(this.indexPath, index, { spaces: 2 });
+        }
+        catch (e) {
+            // If JSON is corrupt, it'll be fixed on next full reindex
+        }
+    }
+    async parseMarkdownFile(filePath, mtimeMs) {
+        try {
+            const content = await fs.readFile(filePath, 'utf8');
+            const relativePath = path.relative(workspaceRoot, filePath);
+            // Use Unified Validation Service for parsing
+            const metadata = validationService.extractMetadata(content);
+            // Extract title (First H1 or filename)
+            let title = metadata.title || '';
+            if (!title) {
+                const h1Match = content.match(/^#\s+(.*)/m);
+                title = h1Match ? h1Match[1] : path.basename(filePath, '.md');
+            }
+            // Generate Excerpt
+            const body = content.replace(/^---[\s\S]*?---/, '').trim();
+            const excerpt = body.split(/\n\s*\n/)[0]
+                .slice(0, 200)
+                .replace(/\r?\n/g, ' ')
+                .trim();
+            // Entity Extraction
+            const mentions = Array.from(body.matchAll(/@(\w+)/g)).map(m => m[1]);
+            const bodyTags = Array.from(body.matchAll(/#(\w+)/g)).map(m => m[1]);
+            const tags = Array.from(new Set([
+                ...(Array.isArray(metadata.Tags) ? metadata.Tags : (Array.isArray(metadata.tags) ? metadata.tags : [])),
+                ...bodyTags
+            ]));
+            return {
+                path: relativePath,
+                title,
+                tags,
+                status: metadata.status || 'active',
+                lastModified: mtimeMs,
+                excerpt,
+                content: body,
+                mentions
+            };
+        }
+        catch (error) {
+            console.error(`Failed to index ${filePath}:`, error);
+            return null;
+        }
+    }
+    /**
+     * Searches the local index.
+     */
+    async search(query) {
+        if (!(await fs.pathExists(this.indexPath))) {
+            return [];
+        }
+        const index = await fs.readJSON(this.indexPath);
+        const lowerQuery = query.toLowerCase();
+        return index.records.filter(record => record.title.toLowerCase().includes(lowerQuery) ||
+            record.tags.some(t => t.toLowerCase().includes(lowerQuery)) ||
+            record.excerpt.toLowerCase().includes(lowerQuery) ||
+            record.path.toLowerCase().includes(lowerQuery));
+    }
+}
+export const globalIndexer = new ContextIndexer();

package/dist/services/database.d.ts

@@ -0,0 +1,30 @@
+import Database from 'better-sqlite3';
+export interface DBRecord {
+    id?: number;
+    path: string;
+    title: string;
+    content: string;
+    excerpt: string;
+    mtime: number;
+    metadata: string;
+}
+export declare class DatabaseService {
+    private db;
+    private dbPath;
+    constructor(workspaceRoot: string);
+    private initializeSchema;
+    upsertDocument(record: DBRecord): {
+        id: number;
+    };
+    upsertVector(docId: number, embedding: Float32Array, provider: string): Database.RunResult;
+    searchHybrid(queryEmbedding: Float32Array, queryText: string, limit?: number): {
+        semanticResults: unknown[];
+        keywordResults: unknown[];
+    };
+    getDocumentByPath(filePath: string): DBRecord | undefined;
+    removeDocument(filePath: string): Database.RunResult;
+    getAllDocuments(): DBRecord[];
+    getVectorForDocument(docId: number): Float32Array<ArrayBufferLike> | undefined;
+    searchSemantic(queryEmbedding: Float32Array, limit?: number): any[];
+    close(): void;
+}

package/dist/services/database.js

@@ -0,0 +1,147 @@
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import path from 'path';
+import fs from 'fs-extra';
+export class DatabaseService {
+    db;
+    dbPath;
+    constructor(workspaceRoot) {
+        const dbDir = path.join(workspaceRoot, '.context-db');
+        fs.ensureDirSync(dbDir);
+        this.dbPath = path.join(dbDir, 'context.db');
+        this.db = new Database(this.dbPath);
+        // Load sqlite-vec extension
+        sqliteVec.load(this.db);
+        this.initializeSchema();
+    }
+    initializeSchema() {
+        // 1. Documents Table
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        path TEXT UNIQUE,
+        title TEXT,
+        content TEXT,
+        excerpt TEXT,
+        mtime INTEGER,
+        metadata TEXT
+      );
+    `);
+        // 2. FTS5 Virtual Table for Keyword Search
+        this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS fts_documents USING fts5(
+        title, 
+        content,
+        content='documents',
+        content_rowid='id'
+      );
+    `);
+        // 3. Triggers to keep FTS in sync
+        this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN
+        INSERT INTO fts_documents(rowid, title, content) VALUES (new.id, new.title, new.content);
+      END;
+      CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN
+        INSERT INTO fts_documents(fts_documents, rowid, title, content) VALUES('delete', old.id, old.title, old.content);
+      END;
+      CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents BEGIN
+        INSERT INTO fts_documents(fts_documents, rowid, title, content) VALUES('delete', old.id, old.title, old.content);
+        INSERT INTO fts_documents(rowid, title, content) VALUES (new.id, new.title, new.content);
+      END;
+    `);
+        // 4. Vector Table (sqlite-vec)
+        // We use a fixed 384 dimensions for the local model 'all-MiniLM-L6-v2'
+        // But Gemini uses 768. To remain hybrid, we support multiple vector tables or variable columns.
+        // For simplicity, we'll use 'vec_documents' with 768 dimensions (Gemini compat)
+        // and pad local vectors if needed, or better: just detect which one we have.
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS vec_documents (
+        id INTEGER PRIMARY KEY REFERENCES documents(id) ON DELETE CASCADE,
+        embedding BLOB, -- Float32Array
+        provider TEXT -- 'local' or 'gemini'
+      );
+    `);
+    }
+    upsertDocument(record) {
+        const stmt = this.db.prepare(`
+      INSERT INTO documents (path, title, content, excerpt, mtime, metadata)
+      VALUES (@path, @title, @content, @excerpt, @mtime, @metadata)
+      ON CONFLICT(path) DO UPDATE SET
+        title = excluded.title,
+        content = excluded.content,
+        excerpt = excluded.excerpt,
+        mtime = excluded.mtime,
+        metadata = excluded.metadata
+      RETURNING id
+    `);
+        return stmt.get(record);
+    }
+    upsertVector(docId, embedding, provider) {
+        const stmt = this.db.prepare(`
+      INSERT INTO vec_documents (id, embedding, provider)
+      VALUES (?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        embedding = excluded.embedding,
+        provider = excluded.provider
+    `);
+        // sqlite-vec expects raw buffer
+        return stmt.run(docId, Buffer.from(embedding.buffer), provider);
+    }
+    searchHybrid(queryEmbedding, queryText, limit = 10) {
+        // This is a simplified hybrid search. 
+        // It finds top semantic matches and top keyword matches.
+        // 1. Semantic Search
+        const semanticStmt = this.db.prepare(`
+      SELECT 
+        d.path, d.title, d.excerpt,
+        vec_distance_cosine(v.embedding, ?) as distance
+      FROM vec_documents v
+      JOIN documents d ON v.id = d.id
+      ORDER BY distance ASC
+      LIMIT ?
+    `);
+        const semanticResults = semanticStmt.all(Buffer.from(queryEmbedding.buffer), limit);
+        // 2. Keyword Search (FTS5)
+        const keywordStmt = this.db.prepare(`
+      SELECT 
+        d.path, d.title, d.excerpt,
+        rank as fts_score
+      FROM fts_documents f
+      JOIN documents d ON f.rowid = d.id
+      WHERE fts_documents MATCH ?
+      ORDER BY rank
+      LIMIT ?
+    `);
+        const keywordResults = keywordStmt.all(queryText, limit);
+        return { semanticResults, keywordResults };
+    }
+    getDocumentByPath(filePath) {
+        return this.db.prepare('SELECT * FROM documents WHERE path = ?').get(filePath);
+    }
+    removeDocument(filePath) {
+        const stmt = this.db.prepare('DELETE FROM documents WHERE path = ?');
+        return stmt.run(filePath);
+    }
+    getAllDocuments() {
+        return this.db.prepare('SELECT * FROM documents').all();
+    }
+    getVectorForDocument(docId) {
+        const row = this.db.prepare('SELECT embedding FROM vec_documents WHERE id = ?').get(docId);
+        return row ? new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 4) : undefined;
+    }
+    searchSemantic(queryEmbedding, limit = 10) {
+        const stmt = this.db.prepare(`
+      SELECT 
+        d.path, d.title, d.excerpt,
+        vec_distance_cosine(v.embedding, ?) as distance
+      FROM vec_documents v
+      JOIN documents d ON v.id = d.id
+      ORDER BY distance ASC
+      LIMIT ?
+    `);
+        return stmt.all(Buffer.from(queryEmbedding.buffer), limit);
+    }
+    close() {
+        this.db.close();
+    }
+}

package/dist/services/embedding.d.ts

@@ -0,0 +1,24 @@
+export interface EmbeddingProvider {
+    name: string;
+    generate(text: string): Promise<Float32Array>;
+    dimension: number;
+}
+export declare class TransformersProvider implements EmbeddingProvider {
+    name: string;
+    dimension: number;
+    private extractor;
+    generate(text: string): Promise<Float32Array>;
+}
+export declare class GeminiProvider implements EmbeddingProvider {
+    name: string;
+    dimension: number;
+    private apiKey;
+    constructor(apiKey: string);
+    generate(text: string): Promise<Float32Array>;
+}
+export declare class EmbeddingService {
+    private provider;
+    constructor(apiKey?: string);
+    getProviderName(): Promise<string>;
+    generate(text: string): Promise<Float32Array>;
+}

package/dist/services/embedding.js

@@ -0,0 +1,55 @@
+import { pipeline } from '@xenova/transformers';
+export class TransformersProvider {
+    name = 'local';
+    dimension = 384;
+    extractor = null;
+    async generate(text) {
+        if (!this.extractor) {
+            this.extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+        }
+        const output = await this.extractor(text, { pooling: 'mean', normalize: true });
+        return new Float32Array(output.data);
+    }
+}
+export class GeminiProvider {
+    name = 'gemini';
+    dimension = 768; // text-embedding-004 standard
+    apiKey;
+    constructor(apiKey) {
+        this.apiKey = apiKey;
+    }
+    async generate(text) {
+        // Implementation for Gemini Embedding API
+        // For now, we provide the structure. 
+        // We'll use a fetch call to the Gemini API endpoint.
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent?key=${this.apiKey}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                content: { parts: [{ text }] }
+            })
+        });
+        const data = await response.json();
+        if (!data.embedding) {
+            throw new Error(`Gemini Embedding Failed: ${JSON.stringify(data)}`);
+        }
+        return new Float32Array(data.embedding.values);
+    }
+}
+export class EmbeddingService {
+    provider;
+    constructor(apiKey) {
+        if (apiKey) {
+            this.provider = new GeminiProvider(apiKey);
+        }
+        else {
+            this.provider = new TransformersProvider();
+        }
+    }
+    async getProviderName() {
+        return this.provider.name;
+    }
+    async generate(text) {
+        return this.provider.generate(text);
+    }
+}

package/dist/services/intelligence.d.ts

@@ -0,0 +1,22 @@
+export interface SearchResult {
+    path: string;
+    title: string;
+    tags: string[];
+    excerpt: string;
+    score?: number;
+    type: 'index' | 'deep' | 'semantic' | 'hybrid';
+}
+export declare class IntelligenceService {
+    private miniSearch;
+    private dbService;
+    private embeddingService;
+    private getIndex;
+    /**
+     * Hybrid Search: Semantic (sqlite-vec) + Keyword (FTS5) -> Lite Index (MiniSearch) -> Grep Fallback
+     */
+    search(query: string, options?: {
+        deep?: boolean;
+    }): Promise<SearchResult[]>;
+    extract(text: string): Promise<string[]>;
+}
+export declare const intelligenceService: IntelligenceService;