@context-os/core 1.0.1 → 1.6.1

package/dist/services/intelligence.js

@@ -0,0 +1,125 @@
+import { exec } from "child_process";
+import { promisify } from "util";
+import MiniSearch from "minisearch";
+import { globalIndexer } from "../indexer.js";
+import { getWorkspaceRoot } from "../context.js";
+import { DatabaseService } from "./database.js";
+import { EmbeddingService } from "./embedding.js";
+const execAsync = promisify(exec);
+export class IntelligenceService {
+    miniSearch = null;
+    dbService = null;
+    embeddingService = null;
+    async getIndex() {
+        if (this.miniSearch)
+            return this.miniSearch;
+        const indexData = await globalIndexer.reindex();
+        this.miniSearch = new MiniSearch({
+            fields: ['title', 'tags', 'excerpt', 'content', 'path'],
+            storeFields: ['path', 'title', 'tags', 'excerpt'],
+            searchOptions: {
+                boost: { title: 2, tags: 1.5 },
+                fuzzy: 0.2,
+                prefix: true
+            }
+        });
+        this.miniSearch.addAll(indexData.records.map((r, i) => ({ ...r, id: i })));
+        return this.miniSearch;
+    }
+    /**
+     * Hybrid Search: Semantic (sqlite-vec) + Keyword (FTS5) -> Lite Index (MiniSearch) -> Grep Fallback
+     */
+    async search(query, options = {}) {
+        // 1. Try Elite Hybrid Search First (SQLite-Vec + FTS5)
+        if (!options.deep) {
+            if (!this.dbService)
+                this.dbService = new DatabaseService(getWorkspaceRoot());
+            if (!this.embeddingService)
+                this.embeddingService = new EmbeddingService(process.env.GEMINI_API_KEY);
+            try {
+                const queryEmbedding = await this.embeddingService.generate(query);
+                const { semanticResults, keywordResults } = this.dbService.searchHybrid(queryEmbedding, query);
+                const results = [];
+                const seenPaths = new Set();
+                // Add Semantic Results
+                semanticResults.forEach((res) => {
+                    results.push({
+                        path: res.path,
+                        title: res.title,
+                        tags: [],
+                        excerpt: res.excerpt,
+                        score: 1 - res.distance,
+                        type: 'semantic'
+                    });
+                    seenPaths.add(res.path);
+                });
+                // Add Keyword Results (if not already seen)
+                keywordResults.forEach((res) => {
+                    if (!seenPaths.has(res.path)) {
+                        results.push({
+                            path: res.path,
+                            title: res.title,
+                            tags: [],
+                            excerpt: res.excerpt,
+                            score: res.fts_score,
+                            type: 'index'
+                        });
+                    }
+                });
+                if (results.length > 0)
+                    return results;
+            }
+            catch (err) {
+                console.error("[IntelligenceService] Hybrid search failed, falling back to Lite:", err);
+            }
+            // 2. Fallback to Lite Index (MiniSearch)
+            const ms = await this.getIndex();
+            const msResults = ms.search(query);
+            if (msResults.length > 0) {
+                return msResults.map(res => ({
+                    path: res.path,
+                    title: res.title,
+                    tags: res.tags,
+                    excerpt: res.excerpt,
+                    score: res.score,
+                    type: 'index'
+                }));
+            }
+        }
+        // 3. Fallback to Deep Scan (Grep)
+        const workspaceRoot = getWorkspaceRoot();
+        const command = `grep -rnIE "${query}" . | head -n 20`;
+        try {
+            const { stdout } = await execAsync(command, {
+                cwd: workspaceRoot,
+                maxBuffer: 10 * 1024 * 1024
+            });
+            if (!stdout)
+                return [];
+            return stdout.split('\n')
+                .filter((line) => line.trim())
+                .map((line) => {
+                const [filePath, ...rest] = line.split(':');
+                const content = rest.join(':').trim();
+                return {
+                    path: filePath,
+                    title: 'Deep Scan Result',
+                    tags: [],
+                    excerpt: content,
+                    type: 'deep'
+                };
+            });
+        }
+        catch (error) {
+            if (error.code === 1)
+                return [];
+            throw error;
+        }
+    }
+    async extract(text) {
+        const mentions = Array.from(text.matchAll(/@(\w+)/g)).map(m => m[1]);
+        const tags = Array.from(text.matchAll(/#(\w+)/g)).map(m => m[1]);
+        return Array.from(new Set([...mentions, ...tags]));
+    }
+}
+export const intelligenceService = new IntelligenceService();

package/dist/services/knowledge-graph.d.ts

@@ -0,0 +1,27 @@
+import { DatabaseService } from "./database.js";
+export interface GraphNode {
+    id: string;
+    label: string;
+    type: 'document' | 'entity' | 'tag';
+    metadata: any;
+}
+export interface GraphEdge {
+    source: string;
+    target: string;
+    type: 'mention' | 'tag' | 'semantic';
+    weight: number;
+}
+export interface WorkspaceGraph {
+    nodes: GraphNode[];
+    edges: GraphEdge[];
+}
+export declare class KnowledgeGraphService {
+    private dbService;
+    constructor(dbService?: DatabaseService);
+    /**
+     * Builds a unified graph of the workspace.
+     * Combines explicit links (@mentions, #tags) and semantic bridges.
+     */
+    getGraph(): Promise<WorkspaceGraph>;
+}
+export declare const knowledgeGraphService: KnowledgeGraphService;

package/dist/services/knowledge-graph.js

@@ -0,0 +1,62 @@
+import { DatabaseService } from "./database.js";
+import { getWorkspaceRoot } from "../context.js";
+export class KnowledgeGraphService {
+    dbService;
+    constructor(dbService) {
+        this.dbService = dbService || new DatabaseService(getWorkspaceRoot());
+    }
+    /**
+     * Builds a unified graph of the workspace.
+     * Combines explicit links (@mentions, #tags) and semantic bridges.
+     */
+    async getGraph() {
+        const nodes = [];
+        const edges = [];
+        const seenNodes = new Set();
+        // 1. Load all documents from the database
+        const docs = this.dbService.getAllDocuments();
+        for (const doc of docs) {
+            // Add Document Node
+            if (!seenNodes.has(doc.path)) {
+                nodes.push({
+                    id: doc.path,
+                    label: doc.title,
+                    type: 'document',
+                    metadata: { excerpt: doc.excerpt }
+                });
+                seenNodes.add(doc.path);
+            }
+            // 2. Extract Explicit Links (Tags/Mentions from metadata JSON)
+            const metadata = JSON.parse(doc.metadata || '[]');
+            metadata.forEach((tag) => {
+                const tagId = `tag:${tag}`;
+                if (!seenNodes.has(tagId)) {
+                    nodes.push({ id: tagId, label: tag, type: 'tag', metadata: {} });
+                    seenNodes.add(tagId);
+                }
+                edges.push({ source: doc.path, target: tagId, type: 'tag', weight: 1.0 });
+            });
+            // 3. Find Semantic Bridges (similarity > 0.85)
+            // We'll query for similar documents for each doc
+            if (doc.id !== undefined) {
+                const vector = this.dbService.getVectorForDocument(doc.id);
+                if (vector) {
+                    const similar = this.dbService.searchSemantic(vector, 10);
+                    similar.forEach((match) => {
+                        // Only bridge if similarity is high and it's not the same doc
+                        if (match.path !== doc.path && (1 - match.distance) > 0.85) {
+                            edges.push({
+                                source: doc.path,
+                                target: match.path,
+                                type: 'semantic',
+                                weight: 1 - match.distance
+                            });
+                        }
+                    });
+                }
+            }
+        }
+        return { nodes, edges };
+    }
+}
+export const knowledgeGraphService = new KnowledgeGraphService();

package/dist/services/sampling.d.ts

@@ -0,0 +1,24 @@
+import { DatabaseService } from "./database.js";
+export interface WorkspacePulse {
+    timestamp: number;
+    healthScore: number;
+    topTags: string[];
+    activeEntities: string[];
+    recentChanges: string[];
+}
+export declare class SamplingService {
+    private dbService;
+    private cache;
+    private CACHE_TTL;
+    constructor(dbService?: DatabaseService);
+    /**
+     * Generates a high-level summary of the entire workspace.
+     * Cached for 5 minutes to prevent expensive re-scans.
+     */
+    getPulse(): Promise<WorkspacePulse>;
+    /**
+     * Public method to invalidate cache manually (used by WatchService)
+     */
+    flushCache(): void;
+}
+export declare const samplingService: SamplingService;

package/dist/services/sampling.js

@@ -0,0 +1,62 @@
+import { DatabaseService } from "./database.js";
+import { getWorkspaceRoot } from "../context.js";
+export class SamplingService {
+    dbService;
+    cache = null;
+    CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+    constructor(dbService) {
+        this.dbService = dbService || new DatabaseService(getWorkspaceRoot());
+    }
+    /**
+     * Generates a high-level summary of the entire workspace.
+     * Cached for 5 minutes to prevent expensive re-scans.
+     */
+    async getPulse() {
+        const now = Date.now();
+        // Check if cache is still valid
+        if (this.cache && this.cache.expiry > now && this.cache.data) {
+            return this.cache.data;
+        }
+        const docs = this.dbService.getAllDocuments();
+        // 1. Calculate Health Score (Basic: % of docs with title and tags)
+        let completeDocs = 0;
+        const tagCounts = {};
+        const entityCounts = {};
+        const recent = [];
+        docs.forEach((doc) => {
+            const tags = JSON.parse(doc.metadata || '[]');
+            if (doc.title && tags.length > 0)
+                completeDocs++;
+            tags.forEach((t) => {
+                tagCounts[t] = (tagCounts[t] || 0) + 1;
+            });
+            // Tracking recent changes (last 5)
+            recent.push(doc.path);
+        });
+        const healthScore = docs.length > 0 ? (completeDocs / docs.length) * 100 : 0;
+        // 2. Extract Top 5 Tags
+        const topTags = Object.entries(tagCounts)
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 5)
+            .map(e => e[0]);
+        const pulse = {
+            timestamp: now,
+            healthScore: Math.round(healthScore),
+            topTags,
+            activeEntities: [], // Placeholder for entity mapping upgrade
+            recentChanges: recent.slice(-5).reverse()
+        };
+        this.cache = {
+            data: pulse,
+            expiry: now + this.CACHE_TTL
+        };
+        return pulse;
+    }
+    /**
+     * Public method to invalidate cache manually (used by WatchService)
+     */
+    flushCache() {
+        this.cache = null;
+    }
+}
+export const samplingService = new SamplingService();

package/dist/services/validation.d.ts

@@ -0,0 +1,31 @@
+export interface ValidationIssue {
+    project: string;
+    file: string;
+    message: string;
+    details?: any;
+}
+export interface ValidationResult {
+    valid: boolean;
+    issues: ValidationIssue[];
+    totalProjects: number;
+}
+export declare class ValidationService {
+    private validFiles;
+    /**
+     * Recursively validates the entire workspace against JSON schemas.
+     */
+    validateWorkspace(): Promise<ValidationResult>;
+    /**
+     * Validates a single file against its schema based on mapping rules.
+     */
+    validateFile(filePath: string): Promise<{
+        valid: boolean;
+        issues: string[];
+    }>;
+    /**
+     * Robustly extract metadata from a markdown file.
+     * Prioritizes Frontmatter, falls back to Section mapping.
+     */
+    extractMetadata(content: string): any;
+}
+export declare const validationService: ValidationService;

package/dist/services/validation.js

@@ -0,0 +1,153 @@
+import fs from "fs-extra";
+import path from "path";
+import AjvModule from "ajv";
+import addFormatsModule from "ajv-formats";
+import fm from "front-matter";
+import { getWorkspaceRoot } from "../context.js";
+const Ajv = AjvModule.default || AjvModule;
+const addFormats = addFormatsModule.default || addFormatsModule;
+const ajv = new Ajv({ allErrors: true });
+addFormats(ajv);
+export class ValidationService {
+    validFiles = [
+        { name: "SOUL.md", schema: "soul.schema.json", required: true },
+        { name: "CONTEXT.md", schema: "context.schema.json", required: true },
+        { name: "memory.md", schema: "memory.schema.json" },
+        { name: "decisions.md", schema: "decision.schema.json" }
+    ];
+    /**
+     * Recursively validates the entire workspace against JSON schemas.
+     */
+    async validateWorkspace() {
+        const workspaceRoot = getWorkspaceRoot();
+        const schemasDir = path.join(workspaceRoot, "packages", "core", "schemas");
+        const projectsDir = path.join(workspaceRoot, "projects");
+        const starterDir = path.join(workspaceRoot, "workspace-starter");
+        const issues = [];
+        const projectPaths = [];
+        // 1. Collect all project paths
+        if (await fs.pathExists(projectsDir)) {
+            const projects = await fs.readdir(projectsDir);
+            for (const p of projects) {
+                const fullPath = path.join(projectsDir, p);
+                if ((await fs.stat(fullPath)).isDirectory()) {
+                    projectPaths.push(fullPath);
+                }
+            }
+        }
+        if (await fs.pathExists(starterDir)) {
+            projectPaths.push(starterDir);
+        }
+        // 2. Validate each project
+        for (const projectPath of projectPaths) {
+            const projectName = path.basename(projectPath);
+            const filesInDir = await fs.readdir(projectPath);
+            for (const config of this.validFiles) {
+                const fileName = filesInDir.find(f => f.toLowerCase() === config.name.toLowerCase());
+                if (!fileName) {
+                    if (config.required) {
+                        issues.push({ project: projectName, file: config.name, message: "Missing required file" });
+                    }
+                    continue;
+                }
+                const filePath = path.join(projectPath, fileName);
+                const schemaPath = path.join(schemasDir, config.schema);
+                try {
+                    const schema = await fs.readJson(schemaPath);
+                    const validate = ajv.compile(schema);
+                    const content = await fs.readFile(filePath, "utf-8");
+                    const data = this.extractMetadata(content);
+                    if (!validate(data)) {
+                        validate.errors?.forEach((err) => {
+                            issues.push({
+                                project: projectName,
+                                file: fileName,
+                                message: `${err.instancePath || 'root'} ${err.message}`,
+                                details: err
+                            });
+                        });
+                    }
+                }
+                catch (error) {
+                    issues.push({ project: projectName, file: fileName, message: `Validation failed: ${error.message}` });
+                }
+            }
+        }
+        return {
+            valid: issues.length === 0,
+            issues,
+            totalProjects: projectPaths.length
+        };
+    }
+    /**
+     * Validates a single file against its schema based on mapping rules.
+     */
+    async validateFile(filePath) {
+        const fileName = path.basename(filePath);
+        const workspaceRoot = getWorkspaceRoot();
+        const schemasDir = path.join(workspaceRoot, "packages", "core", "schemas");
+        // Find matching config
+        const config = this.validFiles.find(v => path.basename(filePath).toLowerCase() === v.name.toLowerCase());
+        if (!config)
+            return { valid: true, issues: [] }; // No schema for this file type
+        const schemaPath = path.join(schemasDir, config.schema);
+        if (!await fs.pathExists(schemaPath))
+            return { valid: true, issues: [] };
+        const schema = await fs.readJson(schemaPath);
+        const validate = ajv.compile(schema);
+        const content = await fs.readFile(filePath, "utf-8");
+        const data = this.extractMetadata(content);
+        const issues = [];
+        if (!validate(data)) {
+            validate.errors?.forEach((err) => {
+                issues.push(`${err.instancePath || 'root'} ${err.message}`);
+            });
+        }
+        return { valid: issues.length === 0, issues };
+    }
+    /**
+     * Robustly extract metadata from a markdown file.
+     * Prioritizes Frontmatter, falls back to Section mapping.
+     */
+    extractMetadata(content) {
+        let data = {};
+        const trimmed = content.trim();
+        // 1. Try Frontmatter
+        try {
+            const parse = fm.default || fm;
+            if (trimmed.startsWith("---")) {
+                const parsed = parse(content);
+                data = parsed.attributes || {};
+            }
+        }
+        catch (e) {
+            // Silently continue
+        }
+        // 2. Try H1 Header as Title if not set
+        if (!data.title || data.title === "") {
+            const h1Match = trimmed.match(/^#\s+(.*)$/m);
+            if (h1Match) {
+                data.title = h1Match[1].trim();
+            }
+        }
+        // 3. Map Sections (## Header)
+        const sections = content.split(/^## /m).slice(1);
+        sections.forEach(s => {
+            const lines = s.split("\n");
+            const title = lines[0].trim();
+            const body = lines.slice(1).join("\n").trim();
+            if (!data[title] || (Array.isArray(data[title]) && data[title].length === 0)) {
+                if (["Core Principles", "Behavioral Rules", "Goals", "Capabilities", "Constraints", "Tags", "Active Tasks", "Backlog"].includes(title)) {
+                    data[title] = body.split("\n")
+                        .map(l => l.replace(/^[-*]\s*/, "").trim())
+                        .filter(l => l.length > 0);
+                }
+                else {
+                    data[title] = body;
+                }
+            }
+        });
+        return data;
+    }
+}
+export const validationService = new ValidationService();

package/dist/services/watch.d.ts

@@ -0,0 +1,16 @@
+export declare class WatchService {
+    private watchers;
+    private debouncers;
+    private DEBOUNCE_MS;
+    /**
+     * Starts watching the allowed buckets for changes.
+     */
+    start(): void;
+    /**
+     * Stops all active watchers.
+     */
+    stop(): void;
+    private debounceChange;
+    private handleEvent;
+}
+export declare const watchService: WatchService;

package/dist/services/watch.js

@@ -0,0 +1,74 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { workspaceRoot, ALLOWED_BUCKETS } from '../context.js';
+import { globalIndexer } from '../indexer.js';
+import { samplingService } from './sampling.js';
+export class WatchService {
+    watchers = [];
+    debouncers = new Map();
+    DEBOUNCE_MS = 300;
+    /**
+     * Starts watching the allowed buckets for changes.
+     */
+    start() {
+        console.log('📡 Starting ContextOS Watch Service...');
+        for (const bucket of ALLOWED_BUCKETS) {
+            const bucketPath = path.join(workspaceRoot, bucket);
+            if (!fs.existsSync(bucketPath))
+                continue;
+            try {
+                // On macOS, recursive: true is supported and very efficient
+                const watcher = fs.watch(bucketPath, { recursive: true }, (eventType, filename) => {
+                    if (!filename || !filename.endsWith('.md'))
+                        return;
+                    const fullPath = path.join(bucketPath, filename);
+                    this.debounceChange(fullPath);
+                });
+                this.watchers.push(watcher);
+                console.log(`  - Watching [${bucket}]`);
+            }
+            catch (error) {
+                console.error(`  - Failed to watch [${bucket}]:`, error);
+            }
+        }
+    }
+    /**
+     * Stops all active watchers.
+     */
+    stop() {
+        for (const watcher of this.watchers) {
+            watcher.close();
+        }
+        this.watchers = [];
+        console.log('🛑 Watch Service stopped.');
+    }
+    debounceChange(filePath) {
+        if (this.debouncers.has(filePath)) {
+            clearTimeout(this.debouncers.get(filePath));
+        }
+        const timeout = setTimeout(async () => {
+            this.debouncers.delete(filePath);
+            await this.handleEvent(filePath);
+        }, this.DEBOUNCE_MS);
+        this.debouncers.set(filePath, timeout);
+    }
+    async handleEvent(filePath) {
+        const relativePath = path.relative(workspaceRoot, filePath);
+        try {
+            if (fs.existsSync(filePath)) {
+                console.log(`📝 Change detected: ${relativePath}`);
+                await globalIndexer.indexFile(filePath);
+            }
+            else {
+                console.log(`🗑️ Deletion detected: ${relativePath}`);
+                await globalIndexer.removeFile(relativePath);
+            }
+            // Invalidate Sampling cache to ensure pulse is fresh
+            samplingService.flushCache();
+        }
+        catch (error) {
+            console.error(`❌ Watch error for ${relativePath}:`, error);
+        }
+    }
+}
+export const watchService = new WatchService();

package/dist/services/workspace.d.ts

@@ -0,0 +1,18 @@
+export declare class WorkspaceService {
+    /**
+     * Synchronizes workspace context and updates metadata.
+     */
+    sync(project?: string, options?: {
+        force?: boolean;
+    }): Promise<{
+        success: boolean;
+        message: string;
+    }>;
+    /**
+     * Initializes a new workspace or project.
+     */
+    init(projectName: string, options?: {
+        template?: string;
+    }): Promise<void>;
+}
+export declare const workspaceService: WorkspaceService;

package/dist/services/workspace.js

@@ -0,0 +1,46 @@
+import fs from "fs-extra";
+import path from "path";
+import { getWorkspaceRoot } from "../context.js";
+import { globalIndexer } from "../indexer.js";
+export class WorkspaceService {
+    /**
+     * Synchronizes workspace context and updates metadata.
+     */
+    async sync(project, options = {}) {
+        const workspaceRoot = getWorkspaceRoot();
+        const date = new Date().toISOString().split("T")[0];
+        if (project) {
+            const projectDir = path.join(workspaceRoot, "projects", project);
+            const memoryPath = path.join(projectDir, "memory.md");
+            if (await fs.pathExists(memoryPath)) {
+                let content = await fs.readFile(memoryPath, "utf-8");
+                const syncMark = `\n> [!NOTE]\n> Last Sync: ${date} ${new Date().toLocaleTimeString()}\n`;
+                if (!content.includes("Last Sync:")) {
+                    await fs.appendFile(memoryPath, syncMark);
+                }
+                else {
+                    content = content.replace(/> \[!NOTE\]\n> Last Sync: .*/, syncMark.trim());
+                    await fs.writeFile(memoryPath, content);
+                }
+            }
+            else {
+                return { success: false, message: `Memory file not found for ${project}` };
+            }
+        }
+        // Auto-refresh the intelligence index as part of sync (v1.4: Incremental by default)
+        await globalIndexer.reindex({ force: options.force });
+        return {
+            success: true,
+            message: project
+                ? `Synced memory for ${project} and refreshed index (incremental).`
+                : `Workspace indexed (${options.force ? 'full re-scan' : 'incremental'}).`
+        };
+    }
+    /**
+     * Initializes a new workspace or project.
+     */
+    async init(projectName, options = {}) {
+        // Logic for init will go here in Phase 2
+    }
+}
+export const workspaceService = new WorkspaceService();

package/dist/tests/core-services.test.d.ts

@@ -0,0 +1 @@
+export {};