@doquflow/server 0.2.0 → 0.4.0

package/dist/tools/generate-dependency-graph.js

@@ -0,0 +1,162 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateDependencyGraph = generateDependencyGraph;
+const node_path_1 = __importDefault(require("node:path"));
+const list_modules_1 = require("./list-modules");
+async function generateDependencyGraph(input) {
+    try {
+        const projectPath = node_path_1.default.resolve(input.project_path);
+        // Scan all modules
+        const listResult = await (0, list_modules_1.listModules)({
+            path: projectPath,
+            extensions: input.extensions,
+        });
+        const modules = listResult.modules ?? [];
+        if (modules.length === 0) {
+            return {
+                nodes: [],
+                edges: [],
+                shared_tables: {},
+                shared_endpoints: {},
+                most_connected: [],
+                summary: "No source files found in the project.",
+            };
+        }
+        // Build id from relative path (normalised for use as node id)
+        const relPath = (filePath) => node_path_1.default.relative(projectPath, filePath).replace(/\\/g, "/");
+        // Build nodes map
+        const nodeMap = new Map();
+        for (const mod of modules) {
+            const id = relPath(mod.path);
+            nodeMap.set(id, {
+                id,
+                label: node_path_1.default.basename(mod.path),
+                language: mod.language,
+                classes: mod.classes.length,
+                functions: mod.functions.length,
+                db_tables: mod.db_tables,
+                endpoints: mod.endpoints,
+            });
+        }
+        // Build import edges: for each module's dependency list, look for matches in node IDs
+        const edges = [];
+        const edgeSet = new Set(); // avoid duplicates
+        for (const mod of modules) {
+            const fromId = relPath(mod.path);
+            for (const dep of mod.dependencies) {
+                // Try to resolve dep to a known node (relative path match or basename match)
+                for (const [nodeId] of nodeMap) {
+                    // Match if the dependency string is contained in the node path
+                    // (e.g., dep = "./user-service" matches "src/user-service.ts")
+                    const depNorm = dep.replace(/^[./]+/, "").replace(/\\/g, "/").toLowerCase();
+                    const nodeNorm = nodeId.replace(/\\/g, "/").toLowerCase();
+                    if (depNorm.length > 2 &&
+                        (nodeNorm.includes(depNorm) || nodeNorm.replace(/\.\w+$/, "").endsWith(depNorm))) {
+                        const key = `${fromId}->${nodeId}`;
+                        if (fromId !== nodeId && !edgeSet.has(key)) {
+                            edgeSet.add(key);
+                            edges.push({ from: fromId, to: nodeId, type: "imports" });
+                        }
+                    }
+                }
+            }
+        }
+        // Shared DB tables
+        const tableToModules = new Map();
+        for (const mod of modules) {
+            for (const table of mod.db_tables) {
+                const key = table.toLowerCase();
+                if (!tableToModules.has(key))
+                    tableToModules.set(key, []);
+                tableToModules.get(key).push(relPath(mod.path));
+            }
+        }
+        const shared_tables = {};
+        for (const [table, mods] of tableToModules) {
+            if (mods.length > 1) {
+                shared_tables[table] = mods;
+                // Add shared_table edges
+                for (let i = 0; i < mods.length; i++) {
+                    for (let j = i + 1; j < mods.length; j++) {
+                        const key = `${mods[i]}<>${mods[j]}:${table}`;
+                        if (!edgeSet.has(key)) {
+                            edgeSet.add(key);
+                            edges.push({ from: mods[i], to: mods[j], type: "shared_table" });
+                        }
+                    }
+                }
+            }
+        }
+        // Shared endpoints
+        const endpointToModules = new Map();
+        for (const mod of modules) {
+            for (const ep of mod.endpoints) {
+                if (!endpointToModules.has(ep))
+                    endpointToModules.set(ep, []);
+                endpointToModules.get(ep).push(relPath(mod.path));
+            }
+        }
+        const shared_endpoints = {};
+        for (const [ep, mods] of endpointToModules) {
+            if (mods.length > 1)
+                shared_endpoints[ep] = mods;
+        }
+        // Most connected nodes (by total edge count)
+        const connectionCount = new Map();
+        for (const edge of edges) {
+            connectionCount.set(edge.from, (connectionCount.get(edge.from) ?? 0) + 1);
+            connectionCount.set(edge.to, (connectionCount.get(edge.to) ?? 0) + 1);
+        }
+        const most_connected = Array.from(connectionCount.entries())
+            .map(([id, count]) => ({
+            id,
+            label: nodeMap.get(id)?.label ?? id,
+            connection_count: count,
+        }))
+            .sort((a, b) => b.connection_count - a.connection_count)
+            .slice(0, 10);
+        // Apply focus filter: if focus is set, only include nodes/edges reachable from it
+        let nodes = Array.from(nodeMap.values());
+        let filteredEdges = edges;
+        if (input.focus) {
+            const focusNorm = input.focus.toLowerCase();
+            const focusId = nodes.find((n) => n.id.toLowerCase().includes(focusNorm))?.id;
+            if (focusId) {
+                const reachable = new Set([focusId]);
+                // BFS: include direct neighbours
+                for (const e of edges) {
+                    if (e.from === focusId)
+                        reachable.add(e.to);
+                    if (e.to === focusId)
+                        reachable.add(e.from);
+                }
+                nodes = nodes.filter((n) => reachable.has(n.id));
+                filteredEdges = edges.filter((e) => reachable.has(e.from) && reachable.has(e.to));
+            }
+        }
+        const sharedTableCount = Object.keys(shared_tables).length;
+        const summary = [
+            `${nodes.length} modules, ${filteredEdges.length} dependencies`,
+            sharedTableCount > 0 ? `${sharedTableCount} shared DB table(s)` : null,
+            most_connected.length > 0
+                ? `Most connected: ${most_connected[0].label} (${most_connected[0].connection_count} links)`
+                : null,
+        ]
+            .filter(Boolean)
+            .join(" · ");
+        return {
+            nodes,
+            edges: filteredEdges,
+            shared_tables,
+            shared_endpoints,
+            most_connected,
+            summary,
+        };
+    }
+    catch (e) {
+        return { error: e?.message ?? String(e) };
+    }
+}

package/dist/tools/get-schema-guidance.js

@@ -0,0 +1,213 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSchemataGuidance = getSchemataGuidance;
+const node_path_1 = __importDefault(require("node:path"));
+const promises_1 = __importDefault(require("node:fs/promises"));
+/**
+ * get_schema_guidance
+ *
+ * Analyzes what documents should exist based on the project schema and current wiki state.
+ * Helps users understand what to create next.
+ *
+ * Input:
+ * - project_path: string
+ * - domain?: string (optional; auto-detected from schema if not provided)
+ *
+ * Output:
+ * - domain: detected domain
+ * - recommended_pages: list of pages that should exist with reasons
+ * - existing_pages: what's already in the wiki
+ * - missing_pages: high-priority missing documents
+ * - recommendations: actionable next steps
+ */
+async function getSchemataGuidance(input) {
+    const projectPath = node_path_1.default.resolve(input.project_path);
+    const docuDir = node_path_1.default.join(projectPath, ".docuflow");
+    const wikiDir = node_path_1.default.join(docuDir, "wiki");
+    const schemaPath = node_path_1.default.join(docuDir, "schema.md");
+    // Read schema to understand domain
+    let domain = input.domain || "General";
+    try {
+        const schemaContent = await promises_1.default.readFile(schemaPath, "utf-8");
+        if (schemaContent.includes("Research"))
+            domain = "Research";
+        else if (schemaContent.includes("Business"))
+            domain = "Business";
+        else if (schemaContent.includes("Architecture"))
+            domain = "Code/Architecture";
+        else if (schemaContent.includes("Personal"))
+            domain = "Personal";
+    }
+    catch {
+        // Use default if schema not readable
+    }
+    // Scan wiki for existing pages
+    const indexPath = node_path_1.default.join(docuDir, "index.md");
+    const indexContent = await promises_1.default
+        .readFile(indexPath, "utf-8")
+        .catch(() => "");
+    // Count pages by category
+    const existingPages = [];
+    const categories = ["entities", "concepts", "syntheses", "timelines"];
+    for (const cat of categories) {
+        const catDir = node_path_1.default.join(wikiDir, cat);
+        try {
+            const files = await promises_1.default.readdir(catDir);
+            for (const file of files.filter((f) => f.endsWith(".md"))) {
+                existingPages.push({
+                    name: file.replace(".md", ""),
+                    category: cat,
+                });
+            }
+        }
+        catch {
+            // Directory may not exist
+        }
+    }
+    // Define recommended pages by domain
+    const recommendedByDomain = {
+        "Code/Architecture": [
+            {
+                name: "architecture_overview",
+                category: "syntheses",
+                suggested_title: "System Architecture Overview",
+                reason: "High-level view of how all components fit together",
+            },
+            {
+                name: "core_patterns",
+                category: "concepts",
+                suggested_title: "Core Architectural Patterns",
+                reason: "Design patterns used throughout the codebase",
+            },
+            {
+                name: "module_dependencies",
+                category: "concepts",
+                suggested_title: "Module Dependencies",
+                reason: "How modules depend on and integrate with each other",
+            },
+            {
+                name: "data_flow",
+                category: "syntheses",
+                suggested_title: "Data Flow Diagram",
+                reason: "How data moves through the system",
+            },
+        ],
+        Research: [
+            {
+                name: "research_overview",
+                category: "syntheses",
+                suggested_title: "Research Domain Overview",
+                reason: "Big picture of the research area",
+            },
+            {
+                name: "key_findings",
+                category: "syntheses",
+                suggested_title: "Key Findings & Synthesis",
+                reason: "Major discoveries and insights",
+            },
+            {
+                name: "contradictions",
+                category: "syntheses",
+                suggested_title: "Areas of Contradiction",
+                reason: "Where researchers disagree",
+            },
+            {
+                name: "open_questions",
+                category: "concepts",
+                suggested_title: "Open Research Questions",
+                reason: "What's still unknown in this domain",
+            },
+        ],
+        Business: [
+            {
+                name: "market_overview",
+                category: "syntheses",
+                suggested_title: "Market Overview",
+                reason: "Big picture of the market and competitive landscape",
+            },
+            {
+                name: "competitive_analysis",
+                category: "syntheses",
+                suggested_title: "Competitive Analysis",
+                reason: "Comparison of key competitors",
+            },
+            {
+                name: "opportunities",
+                category: "syntheses",
+                suggested_title: "Market Opportunities",
+                reason: "Gaps and growth areas",
+            },
+            {
+                name: "risks",
+                category: "concepts",
+                suggested_title: "Market Risks",
+                reason: "Threats and challenges",
+            },
+        ],
+        Personal: [
+            {
+                name: "learning_goals",
+                category: "concepts",
+                suggested_title: "Learning Goals",
+                reason: "What you want to learn in this domain",
+            },
+            {
+                name: "key_insights",
+                category: "syntheses",
+                suggested_title: "Key Personal Insights",
+                reason: "Major learnings and takeaways",
+            },
+            {
+                name: "action_items",
+                category: "concepts",
+                suggested_title: "Action Items & Next Steps",
+                reason: "What to do with this knowledge",
+            },
+            {
+                name: "resources",
+                category: "concepts",
+                suggested_title: "Key Resources",
+                reason: "Important links, books, people",
+            },
+        ],
+    };
+    const recommended = recommendedByDomain[domain] || recommendedByDomain.General;
+    // Find missing pages
+    const missingPages = [];
+    for (const rec of recommended) {
+        const found = existingPages.find((p) => p.name === rec.name);
+        if (!found) {
+            missingPages.push(rec.suggested_title);
+        }
+    }
+    // Generate recommendations
+    const recommendations = [];
+    if (existingPages.length === 0) {
+        recommendations.push("🌱 Start by ingesting your first source");
+        recommendations.push("📝 Create an overview/synthesis page");
+    }
+    else if (existingPages.length < 10) {
+        recommendations.push("📚 Add more sources to deepen understanding");
+        if (missingPages.length > 0) {
+            recommendations.push(`⚠️ Consider creating: ${missingPages[0]}`);
+        }
+    }
+    else if (existingPages.length > 50) {
+        recommendations.push("✅ You have a solid wiki foundation");
+        recommendations.push("🔍 Run lint_wiki to health-check for gaps");
+        recommendations.push("🔗 Verify cross-references are accurate");
+    }
+    if (missingPages.length > 2) {
+        recommendations.push(`💡 Biggest gap: ${missingPages[0]} could improve wiki significantly`);
+    }
+    return {
+        domain,
+        recommended_pages: recommended,
+        existing_pages: existingPages,
+        missing_pages: missingPages,
+        recommendations,
+    };
+}

package/dist/tools/ingest-source.js

@@ -0,0 +1,252 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ingestSource = ingestSource;
+const node_path_1 = __importDefault(require("node:path"));
+const filesystem_1 = require("../filesystem");
+/**
+ * Find the first paragraph in the source that mentions the given name.
+ * Returns cleaned text (stripped of markdown syntax), up to 400 chars.
+ */
+function findContextParagraph(content, name) {
+    const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = new RegExp(`\\b${escaped}\\b`, "i");
+    const paragraphs = content.split(/\n{2,}/);
+    for (const para of paragraphs) {
+        if (!re.test(para))
+            continue;
+        const clean = para
+            .replace(/^#+\s*/gm, "")
+            .replace(/\*\*?([^*]+)\*\*?/g, "$1")
+            .replace(/`([^`]+)`/g, "$1")
+            .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
+            .trim();
+        if (clean.length > 20)
+            return clean.substring(0, 400);
+    }
+    return null;
+}
+/**
+ * Simple markdown-based entity extraction.
+ * Looks for patterns like:
+ * - **Entity:** or ### Entity Headers
+ * - Lists of concepts
+ * - Tool/component mentions
+ */
+function extractFromMarkdown(content) {
+    const lines = content.split("\n");
+    const entities = [];
+    const concepts = new Set();
+    const relationships = [];
+    // Extract first 500 chars as summary (non-empty)
+    const summary = content
+        .split("\n")
+        .filter((l) => l.trim().length > 0 && !l.startsWith("#") && !l.startsWith("```") && !l.startsWith("["))
+        .slice(0, 3)
+        .join(" ")
+        .substring(0, 500);
+    // Find headers (entities/concepts)
+    for (const line of lines) {
+        // ### Header → entity/concept
+        if (line.match(/^###\s+/)) {
+            const header = line.replace(/^###\s+/, "").trim();
+            if (header && !header.startsWith("[") && !header.startsWith("{")) {
+                entities.push({ name: header, type: "concept" });
+            }
+        }
+        // **bold text** → potential entity (but not arrays or JSON)
+        const boldMatches = line.matchAll(/\*\*([^*]+)\*\*/g);
+        for (const match of boldMatches) {
+            const text = match[1].trim();
+            // Skip if it looks like JSON or code
+            if (text.length > 2 &&
+                text.length < 100 &&
+                !text.includes("[") &&
+                !text.includes("{") &&
+                !text.includes('"') &&
+                !text.includes("`")) {
+                entities.push({ name: text, type: "entity" });
+            }
+        }
+    }
+    // Find relationship patterns like "X integrates Y" or "X depends on Y"
+    const relWords = [
+        { word: "integrates", rel: "integrates" },
+        { word: "depends on", rel: "depends_on" },
+        { word: "extends", rel: "extends" },
+        { word: "uses", rel: "uses" },
+        { word: "manages", rel: "manages" },
+    ];
+    for (const line of lines) {
+        for (const { word, rel } of relWords) {
+            const regex = new RegExp(`([A-Z][A-Za-z0-9_]+)\\s+${word}\\s+([A-Z][A-Za-z0-9_]+)`, "gi");
+            const matches = line.matchAll(regex);
+            for (const match of matches) {
+                relationships.push({ from: match[1], to: match[2], relation: rel });
+            }
+        }
+    }
+    // Collect concepts from lines containing "concept:", "pattern:", etc (skip arrays)
+    const conceptLines = lines.filter((l) => /concept|pattern|principle|approach/i.test(l) && !l.startsWith("[") && !l.startsWith("{"));
+    for (const line of conceptLines) {
+        const conceptMatch = line.match(/:\s*([^[\]{}"`.]+?)(?:\.|$)/);
+        if (conceptMatch) {
+            const concept = conceptMatch[1].trim();
+            if (concept && concept.length < 100) {
+                concepts.add(concept);
+            }
+        }
+    }
+    return {
+        summary,
+        entities: Array.from(new Set(entities.map((e) => JSON.stringify(e)))).map((s) => JSON.parse(s)),
+        concepts: Array.from(concepts),
+        relationships,
+    };
+}
+/**
+ * Convert a source document into a collection of wiki pages.
+ * For each distinct entity/concept, create a page.
+ */
+function generateWikiPages(sourceId, sourceTitle, extracted, sourceContent) {
+    const now = new Date().toISOString();
+    const pages = [];
+    // Create summary page (synthesis)
+    const summaryId = `source_${sourceId.replace(/[^a-z0-9]/gi, "_").toLowerCase()}`;
+    const summaryPage = {
+        id: summaryId,
+        title: `Source: ${sourceTitle}`,
+        category: "synthesis",
+        content: `# ${sourceTitle}\n\n${extracted.summary}\n\n## Key Entities\n\n${extracted.entities.map((e) => `- **${e.name}** (${e.type})`).join("\n")}`,
+        frontmatter: {
+            created_at: now,
+            updated_at: now,
+            sources: [sourceId],
+            tags: ["source", "ingested"],
+            inbound_links: [],
+            outbound_links: extracted.entities.map((e) => `entity_${e.name.replace(/[^a-z0-9]/gi, "_").toLowerCase()}`),
+        },
+    };
+    pages.push(summaryPage);
+    // Create entity pages
+    for (const entity of extracted.entities) {
+        const entityId = `entity_${entity.name.replace(/[^a-z0-9]/gi, "_").toLowerCase()}`;
+        const contextPara = findContextParagraph(sourceContent, entity.name);
+        const overview = contextPara
+            ? `${contextPara}\n\n*Introduced in: ${sourceTitle}*`
+            : `Introduced in: ${sourceTitle}`;
+        const entityPage = {
+            id: entityId,
+            title: entity.name,
+            category: "entity",
+            content: `# ${entity.name}\n\n**Type:** ${entity.type}\n\n## Overview\n\n${overview}`,
+            frontmatter: {
+                created_at: now,
+                updated_at: now,
+                sources: [sourceId],
+                tags: [entity.type],
+                inbound_links: [summaryId],
+                outbound_links: [],
+            },
+        };
+        pages.push(entityPage);
+    }
+    // Create concept pages
+    for (const concept of extracted.concepts) {
+        const conceptId = `concept_${concept.replace(/[^a-z0-9]/gi, "_").toLowerCase()}`;
+        const contextPara = findContextParagraph(sourceContent, concept);
+        const definition = contextPara
+            ? `${contextPara}\n\n*Introduced in: ${sourceTitle}*`
+            : `To be expanded with additional sources.\n\n*Introduced in: ${sourceTitle}*`;
+        const conceptPage = {
+            id: conceptId,
+            title: concept,
+            category: "concept",
+            content: `# ${concept}\n\n## Definition\n\n${definition}`,
+            frontmatter: {
+                created_at: now,
+                updated_at: now,
+                sources: [sourceId],
+                tags: ["concept"],
+                inbound_links: [summaryId],
+                outbound_links: [],
+            },
+        };
+        pages.push(conceptPage);
+    }
+    return pages;
+}
+async function ingestSource(input) {
+    try {
+        const projectPath = node_path_1.default.resolve(input.project_path);
+        const docuDir = node_path_1.default.join(projectPath, ".docuflow");
+        const sourcesDir = node_path_1.default.join(docuDir, "sources");
+        const wikiDir = node_path_1.default.join(docuDir, "wiki");
+        // Read source file
+        const sourceFile = node_path_1.default.join(sourcesDir, input.source_filename);
+        const fileRead = await (0, filesystem_1.safeReadFile)(sourceFile);
+        if (fileRead.error) {
+            return {
+                source_id: input.source_filename,
+                summary: `Error reading source: ${fileRead.error}`,
+                pages_created: [],
+                pages_updated: [],
+                entities_discovered: [],
+                contradictions: [],
+            };
+        }
+        const sourceContent = fileRead.content ?? "";
+        const sourceTitle = input.source_filename.replace(".md", "");
+        // Extract information
+        const extracted = extractFromMarkdown(sourceContent);
+        // Generate wiki pages
+        const wikiPages = generateWikiPages(sourceTitle, sourceTitle, extracted, sourceContent);
+        // Write all pages
+        const pagesCreated = [];
+        for (const page of wikiPages) {
+            // Determine category subdirectory - use correct plural forms
+            const pluralMap = {
+                entity: "entities",
+                concept: "concepts",
+                timeline: "timelines",
+                synthesis: "syntheses",
+            };
+            const categoryDir = node_path_1.default.join(wikiDir, pluralMap[page.category] || page.category + "s");
+            await (0, filesystem_1.ensureDir)(categoryDir);
+            // Create page file with frontmatter
+            const frontmatterYaml = `---
+created_at: ${page.frontmatter.created_at}
+updated_at: ${page.frontmatter.updated_at}
+sources: ${JSON.stringify(page.frontmatter.sources)}
+tags: ${JSON.stringify(page.frontmatter.tags)}
+inbound_links: ${JSON.stringify(page.frontmatter.inbound_links)}
+outbound_links: ${JSON.stringify(page.frontmatter.outbound_links)}
+---
+`;
+            const pageContent = frontmatterYaml + "\n" + page.content;
+            const pageFile = node_path_1.default.join(categoryDir, `${page.id}.md`);
+            await (0, filesystem_1.writeFileAtomic)(pageFile, pageContent);
+            pagesCreated.push(page.id);
+        }
+        return {
+            source_id: sourceTitle,
+            summary: extracted.summary,
+            pages_created: pagesCreated,
+            pages_updated: [],
+            entities_discovered: extracted.entities.map((e) => e.name),
+            contradictions: [],
+        };
+    }
+    catch (e) {
+        return {
+            source_id: input.source_filename,
+            summary: `Ingest failed: ${e?.message ?? String(e)}`,
+            pages_created: [],
+            pages_updated: [],
+            entities_discovered: [],
+            contradictions: [],
+        };
+    }
+}